SPARK安装

下载scala-2.10.6包解压到指定目录
http://downloads.typesafe.com/scala/2.11.4/scala-2.11.4.tgz

#SCALA VARIABLES START
export SCALA_HOME=/usr/local/scala-2.10.6
export PATH=$PATH:$SCALA_HOME/bin
#SCALA VARIABLES END1234

2.下载Spark-1.5.2包解压到指定目录

#SPARK VARIABLES START
export SPARK_HOME=/usr/local/spark-1.5.2
export PATH=$PATH:$SPARK_HOME/bin
#SPARK VARIABLES END1234

3.配置spark-env.sh

export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_66
export SCALA_HOME=/usr/local/scala-2.10.6
export HADOOP_HOME=/usr/local/hadoop-2.6.0
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_MASTER_IP=10.9.2.100
export SPARK_WORKER_MEMORY=1024m
export SPARK_LOCAL_DIR="/usr/local/spark-1.5.2/tmp"

4.启动集群(机器ssh端口改变时)
启动主节点：sbin/start-master.sh
启动从节点：sbin/start-slave.sh 10.9.2.100:7077
5.验证
#本地模式两线程运行
./bin/run-example SparkPi 10 --master local[2]

#Spark Standalone 集群模式运行
./bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://10.9.2.100:7077 lib/spark-examples-1.5.2-hadoop2.6.0.jar 100

#Spark on YARN 集群上 yarn-cluster 模式运行
./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn-cluster lib/spark-examples*.jar 10

spark-submit --master spark://10.107.12.10:7077 --class org.apache.spark.examples.SparkPi --name Spark-Pi /root/spark_sdk/spark-1.5.1-bin-hadoop2.6/lib/spark-examples-1.5.1-hadoop2.6.0.jar