大数据学习37：DataFrame集合减操作例子

df1.txt:
1   wxk 44
2   panda   55
3   monkey  66
4   tiger   33

df2.txt:
2   china
3   usa

+---+-----+-----+
| id| name|score|
+---+-----+-----+
|  1|  wxk|   44|
|  4|tiger|   33|
+---+-----+-----+

package zuoye11_22
import org.apache.spark.sql.SparkSession
/**
* 用来将DF1 - DF2 ,对DF1 进行一些过滤
* 2017.11.12
*/
object Df1MinuseDf2 {
def main(args: Array[String]): Unit = {
//1、拿到SparkSession
val spark = SparkSession.builder().master( "local[2]" ).appName( "Df1MinuseDf2App" ).getOrCreate()
runUDFdf1Minusedf2( spark )
spark.stop()
}

/**
* df1 - df2 的功能
* 采用不同方式进行操作，进行对比
* 采用case class 的方式 不建议这么用
* * @param spark
*/
private def runUDFdf1Minusedf2(spark: SparkSession): Unit = {
import spark.implicits._
val DF1 = spark.sparkContext.textFile( "D:/test/df1.txt" )
.map( _.split( "\t" ) ).map( x => {
DF1_structure( x( 0 ).toInt, x( 1 ), x( 2 ) )
} ).toDF()
val DF2 = spark.sparkContext.textFile( "D:/test/df2.txt" )
.map( _.split( "\t" ) ).map( x => {
DF2_structure( x( 0 ).toInt, x( 1 ) )
} ).toDF()
//查看DF的结构
//    DF1.printSchema()
//    DF2.printSchema()

//方法一：
//进行集合减操作
//    DF1.createOrReplaceTempView( "DF1_table" )
//    DF2.createOrReplaceTempView( "DF2_table" )
//    spark.sql( " select A.id from DF1_table A   " ).show
//    spark.sql( " select B.id from DF2_table B  " ).show
//用in 的方式,先得到一个只有id的集合，再用in去匹配
//spark.sql( " select * from DF1_table C where C.id  in (select A.id from DF1_table A minus select B.id from DF2_table B)  " ).show
//用leftjoin 方式
//    spark.sql( "select dd.id,dd.name,dd.score from " +
//    "(select * from " +
//    "(select aa.*,bb.id as tmp  from DF1_table aa left join DF2_table bb on aa.id=bb.id) cc" +
//    " where cc.tmp is null) dd " ).show()

//方法二：
//采用 DataFrame 相关算子,这种方式不需要建临时表，建议使用
DF1.join(DF2,Array("id"),"left_outer").filter("loc is null").select("id","name","score").show()

}
//采用 case class 方式去建 DataFrame
case class DF1_structure(id: Int, name: String, score: String)
case class DF2_structure(id: Int, loc: String)

}