hadoop spark 安装 过程中的临时配置（伪分布式）

个人玩

伪分布式足够啦。

export JAVA_HOME=/home/vincent/jdk1.8.0_11

export JRE_HOME=${JAVA_HOME}/jre  

export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib  

export PATH=${JAVA_HOME}/bin:$PATH

#set scala environment

export SCALA_HOME=/home/vincent/scala-2.11.12

export PATH=$SCALA_HOME/bin:$PATH

# Hadoop Environment Variables

export HADOOP_HOME=/home/vincent/hadoop-2.7.5

export HADOOP_INSTALL=$HADOOP_HOME

export HADOOP_MAPRED_HOME=$HADOOP_HOME

export HADOOP_COMMON_HOME=$HADOOP_HOME

export HADOOP_HDFS_HOME=$HADOOP_HOME

export YARN_HOME=$HADOOP_HOME

export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin

# hadoop-env.sh

export JAVA_HOME=/home/vincent/jdk1.8.0_11

export HADOOP_CONF_DIR=/home/vincent/hadoop-2.7.5/etc/hadoop/

#core-site.xml

<property>  

    <name>fs.default.name</name>  

    <value>hdfs://localhost:9000</value>  

</property>  

<property>  

    <name>hadoop.tmp.dir</name>  

    <value>/home/vincent/hadoop-2.7.5/tmp</value>  

</property>  

## hdfs-site.xml

<!--设置副本数1，不写默认是3份-->  

<property>

    <name>dfs.replication</name>  

    <value>1</value>  

</property>

<property>

    <name>dfs.namenode.name.dir</name>

    <value>file:/home/vincent/hadoop-2.7.5/hdfs/name</value>

</property>

<property>

    <name>dfs.datanode.data.dir</name>

    <value>file:/home/vincent/hadoop-2.7.5/hdfs/data</value>

</property>

##mapred-site.xml  

<property>  

   <name>mapreduce.framework.name</name>  

   <value>yarn</value>  

</property>  

##yarn-site.xml

<property>  

  <name>yarn.nodemanager.aux-services </name>  

  <value>mapreduce_shuffle</value>  

</property> 

##.bashrc

export SPARK_HOME=/home/vincent/spark-2.2.1-bin-hadoop2.7

export PATH=$PATH:$SPARK_HOME/bin

## spark-env.sh

export JAVA_HOME=/home/vincent/jdk1.8.0_11

export SCALA_HOME=/home/vincent/scala-2.11.12

export HADOOP_HOME=/home/vincent/hadoop-2.7.5

export HADOOP_CONF_DIR=/home/vincent/hadoop-2.7.5/etc/hadoop

export SPARK_MASTER_IP=192.168.3.108

export SPARK_WORKER_MEMORY=1g

export SPARK_WORKER_CORES=2
export SPARK_WORKER_INSTANCES=1

测试：

var srcFile = sc.textFile("/testdata/test.csv")

var a = srcFile.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey(_+_)

a.map(word=>(word._2,word._1)).sortByKey(false).map(word=>(word._2,word._1)).take(50).foreach(println)