spark操作hive简单代码

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

/**
* Created by zengxiaosen on 16/9/23.
*/
object hiveoperation {

def main(args: Array[String]): Unit = {
val sparkConf = new SparkConf().setAppName("hive").setMaster("local")
val ss = SparkSession.builder()
.enableHiveSupport()
.config(sparkConf)
.getOrCreate()
import ss.implicits._

val date = "2015-08-28" //通常通过参数传过来

/*
在hive表中
desc track_log
出现:
id string
url string
referer string
keyword string
type string
guid string
pageid string
moduleid string
linkid string
attachedinfo string
sessionid string
trackeru string
cookie string
ordercode string
tracktime string
enduserid string
firstlink string
sessionviewno string
productid string
curmerchantid string
provinceid string
cityid string
ds string
hour string

#Partition information
#col_name data_type comment
ds        string
hour      string
*/

val sqlStr =
s"""
|insert overwrite into daily_visit partition (date='$date')    //日期,通常是通过参数传进来的
|select date,count(distinct guid) uv,sum(pv) pv,
|count(case when pv>=2 then sessionid else null end) second_num,
|count(sessionid) visits from
|(select ds date, sessionid, max(guid) guid, count(url) pv from tracklog and hour='18'
|group by ds,sessionid) a
|group by date
""".stripMargin

println("执行中。。。"+sqlStr)

//返回dataframe,即dataset
val rdd = ss.sql(sqlStr)
rdd.rdd.foreach(println)

/*
把hive-site.xml放到工程里,执行无法直接连上hive!!!
spark通过thrift服务,访问hive元数据库
在pom中加入
<property>
<name>hive.metastore.uris</name>
<value>thrift://master:9083</value>
</property>
在idea上从来没有实现过,所以还是打包去集群去运行把!
原因是idea无法连上thrift://master:9083
*/

/*
结果落到hive表
在hive中建表:
create table daily_visit(
//date string因为是分区表,所以这个日期就不要里
uv bigint,
pv bigint,
second_num bigint,
visits bigint) partitioned by(date string)

show tables
desc daily_visit

*/

}

}