spark 连接 mysql 数据库
2016-01-28 15:54
736 查看
在所有master和slave上也要在spark/conf/spark-conf.sh里面设置driver的classpath,解决编译找不到driver的问题
http://www.iteblog.com/archives/1300
类似这样:
spark-sql driver要到1.4以上
sbt:
name := "hoursAvg"
version := "1.0"
val apacheSpark = "org.apache.spark" %% "spark-core" % "1.2.0"
val apacheSQL = "mysql" % "mysql-connector-java" % "5.1.37"
val apacheSSQL = "org.apache.spark" % "spark-sql_2.10" % "1.4.0"
lazy val commonSettings = Seq(
organization := "com.gws",
version := "0.1.0",
scalaVersion := "2.10.4"
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
name := "hoursAvg",
libraryDependencies ++= Seq (
apacheSQL,
apacheSSQL,
apacheSpark.
exclude("com.esotericsoftware.kryo", "kryo") .
exclude("javax.activation", "activation") .
exclude("commons-logging", "commons-logging") .
exclude("commons-collections", "commons-collections") .
exclude("org.eclipse.jetty.orbit", "javax.transaction") .
exclude("org.eclipse.jetty.orbit", "javax.servlet") .
exclude("org.eclipse.jetty.orbit", "javax.mail.glassfish") .
exclude("org.eclipse.jetty.orbit", "javax.activation")
)
)
scala:
import java.sql.{PreparedStatement, Connection, DriverManager}
import java.util.Properties
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}
import com.mysql.jdbc.Driver
import java.util.Properties
object hoursAvg {
val host = "IP"
val database = "database name"
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("RDDRelation").setMaster("spark://Master.Hadoop:7077")
val sc = new SparkContext(sparkConf)
val sqlContext = new SQLContext(sc)
Class.forName("com.mysql.jdbc.Driver")
val url = "jdbc:mysql://IP:3306/databasename"
val prop = new Properties()
prop.setProperty("user","user name")
prop.setProperty("password","password")
val jdbcDF = sqlContext.read.jdbc(url,"AccountStats",prop)
jdbcDF.show(50)
}
}
http://www.iteblog.com/archives/1300
类似这样:
export
SPARK_CLASSPATH=$SPARK_CLASSPATH:/iteblog/com/mysql-connector-java-5.1.3
spark-sql driver要到1.4以上
sbt:
name := "hoursAvg"
version := "1.0"
val apacheSpark = "org.apache.spark" %% "spark-core" % "1.2.0"
val apacheSQL = "mysql" % "mysql-connector-java" % "5.1.37"
val apacheSSQL = "org.apache.spark" % "spark-sql_2.10" % "1.4.0"
lazy val commonSettings = Seq(
organization := "com.gws",
version := "0.1.0",
scalaVersion := "2.10.4"
)
lazy val root = (project in file(".")).
settings(commonSettings: _*).
settings(
name := "hoursAvg",
libraryDependencies ++= Seq (
apacheSQL,
apacheSSQL,
apacheSpark.
exclude("com.esotericsoftware.kryo", "kryo") .
exclude("javax.activation", "activation") .
exclude("commons-logging", "commons-logging") .
exclude("commons-collections", "commons-collections") .
exclude("org.eclipse.jetty.orbit", "javax.transaction") .
exclude("org.eclipse.jetty.orbit", "javax.servlet") .
exclude("org.eclipse.jetty.orbit", "javax.mail.glassfish") .
exclude("org.eclipse.jetty.orbit", "javax.activation")
)
)
scala:
import java.sql.{PreparedStatement, Connection, DriverManager}
import java.util.Properties
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}
import com.mysql.jdbc.Driver
import java.util.Properties
object hoursAvg {
val host = "IP"
val database = "database name"
def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("RDDRelation").setMaster("spark://Master.Hadoop:7077")
val sc = new SparkContext(sparkConf)
val sqlContext = new SQLContext(sc)
Class.forName("com.mysql.jdbc.Driver")
val url = "jdbc:mysql://IP:3306/databasename"
val prop = new Properties()
prop.setProperty("user","user name")
prop.setProperty("password","password")
val jdbcDF = sqlContext.read.jdbc(url,"AccountStats",prop)
jdbcDF.show(50)
}
}
相关文章推荐
- 21-修改表给mysql添加外键
- mysql json字段的使用与意义
- MySQL两个表联合查询并按时间排序
- Mysql 数据库缓存cache功能
- MySQL数据库性能优化之缓存参数优化
- mysql 区分大小写查询
- 001.mysql安装(lnmp)
- mysql中如何把字符串转换成日期类型
- mysql读取自动增长主键值
- Mysql事件学习
- MySQL恢复中的几个问题解决方法
- MySQL判断表和字段不存在再添加
- MysqL的root用户不允许远程连接
- mysql存储过程
- mysql中日期比较大小方法详解
- MySql 申明变量以及赋值
- MySQL的if,case语句使用总结
- MySQL中常用字符串函数
- MySql字符转义
- MySQL查看连接数