lkl风控.逻辑回归分析模型测试代码spark1.6

/**
* Created by lkl on 2017/10/31.
*/
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.SparkConf
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkContext
import org.apache.spark.mllib.classification.{LogisticRegressionWithLBFGS, LogisticRegressionModel}
import org.apache.spark.mllib.evaluation.MulticlassMetrics
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.sql.SQLContext
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
object logisticregression {

def main(args: Array[String]): Unit = {
val conf = new SparkConf().setAppName("test") //setMaster("spark://192.168.0.37:7077")
val sc = new SparkContext(conf)
val sqlContext = new SQLContext(sc)
val hc = new HiveContext(sc)
val data2 = hc.sql("select * from  fin_tec.uvcy2")
val data = data2.map{ row => val arr = new ArrayBuffer[Double]()
// 第一个变量为身份证号，第二个变量为是否逾期，先过滤
for(i <- 2 until row.size){
if(row.isNullAt(i)){
arr += 0.0}
else if(row.get(i).isInstanceOf[Double])
arr += row.getDouble(i)
else if(row.get(i).isInstanceOf[Long])
arr += row.getLong(i).toDouble
else if(row.get(i).isInstanceOf[String])
arr += row.getString(i).toDouble}
LabeledPoint(row.getDouble(1), Vectors.dense(arr.toArray))}
val splits = data.randomSplit(Array(0.7, 0.3))
val (trainingData, testData) = (splits(0), splits(1))
val training = splits(0).cache()
val test = splits(1)
// Run training algorithm to build the model
val model = new LogisticRegressionWithLBFGS().setNumClasses(10).run(training)

// Compute raw scores on the test set.
val predictionAndLabels = test.map { case LabeledPoint(label, features) =>
val prediction = model.predict(features)
(prediction, label)
}

// Get evaluation metrics.
val metrics = new MulticlassMetrics(predictionAndLabels)
val precision = metrics.precision
println("Precision = " + precision)

// Save and load model
model.save(sc, "myModelPath")
val sameModel = LogisticRegressionModel.load(sc, "myModelPath")

}

}