hadoop-2.0.0-cdh4.2.1安装手册

1. 角色分配：

ocdata16: 10.1.253.26 :NameNode JournalNode Zookeeper DFSZKFailoverController

ocdata17: 10.1.253.27 :NameNode JournalNode Zookeeper DFSZKFailoverController

ocdata18: 10.1.253.28 :DataNode JournalNode Zookeeper
ocdata19: 10.1.253.29 :DataNode

2. 添加SSH互信

登录10.1.253.26\10.1.253.27\10.1.253.28\10.1.253.29

ssh-keygen -t rsa -P ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod go-w $HOME $HOME/.ssh
chmod 600 $HOME/.ssh/authorized_keys
chown `whoami` $HOME/.ssh/authorized_keys

登录10.1.253.26,执行

ssh stream_dev@ocdata17 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
ssh stream_dev@ocdata18 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
ssh stream_dev@ocdata19 cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

scp ~/.ssh/authorized_keys stream_dev@ocdata17:~/.ssh/authorized_keys
scp ~/.ssh/authorized_keys stream_dev@ocdata18:~/.ssh/authorized_keys
scp ~/.ssh/authorized_keys stream_dev@ocdata19:~/.ssh/authorized_keys

3. 安装

下载安装包

wget http://archive.cloudera.com/cdh4/cdh/4/zookeeper-3.4.5-cdh4.2.1.tar.gz wget http://archive.cloudera.com/cdh4/cdh/4/hadoop-2.0.0-cdh4.2.1.tar.gz
解压，配置

配置zookeeper

cd /home/stream_dev/shaoaq/zookeeper-3.4.5-cdh4.2.1/conf
cp zoo_sample.cfg zoo.cfg
修改：
dataDir=/home/stream_dev/shaoaq/zookeeper
clientPort=2183
server.16=ocdata16:2898:3898
server.17=ocdata17:2898:3898
server.18=ocdata18:2898:3898

在 ocdata16\ocdata17\ocdata18上，
mkdir /home/stream_dev/shaoaq/zookeeper,
然后vim /home/stream_dev/shaoaq/zookeeper/myid,分别输入16，17，18

配置hadoop

hdfs-site.xml

<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>
<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>ocdata16:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>ocdata17:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn1</name>
<value>ocdata16:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>ocdata17:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://ocdata16:9485;ocdata17:9485;ocdata18:9485/mycluster</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.mycluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>10000</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/stream_dev/shaoaq/journal/node/local/data</value>
</property>
<property>
<name>dfs.journalnode.http-address</name>
<value>0.0.0.0:9480</value>
</property>
<property>
<name>dfs.journalnode.rpc-address</name>
<value>0.0.0.0:9485</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>ocdata16:2183,ocdata17:2183,ocdata18:2183</value>
</property>

core-site.xml

<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>

slaves

ocdata16
ocdata17
ocdata18
ocdata19

hadoop-env.sh

export JAVA_HOME=/home/stream_dev/stream/soft/jdk1.7.0_21

新建sbin/start-dfs-journalnode.sh

#!/usr/bin/env bash

bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`

DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh

#---------------------------------------------------------
# quorumjournal nodes (if any)

SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)

case "$SHARED_EDITS_DIR" in
qjournal://*)
JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
echo "Starting journal nodes [$JOURNAL_NODES]"
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$JOURNAL_NODES" \
--script "$bin/hdfs" start journalnode ;;
esac

新建sbin/stop-dfs-journalnode.sh

#!/usr/bin/env bash

bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`

DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh

#---------------------------------------------------------
# quorumjournal nodes (if any)

SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-)

case "$SHARED_EDITS_DIR" in
qjournal://*)
JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
echo "Stopping journal nodes [$JOURNAL_NODES]"
"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \
--config "$HADOOP_CONF_DIR" \
--hostnames "$JOURNAL_NODES" \
--script "$bin/hdfs" stop journalnode ;;
esac

分发到其他机器

scp -r ~/shaoaq stream_dev@ocdata17:~/
scp -r ~/shaoaq stream_dev@ocdata18:~/
scp -r ~/shaoaq stream_dev@ocdata19:~/

4. 初始化

启动HDFS

登录ocdata16,执行

./hdfs namenode -format
./hdfs namenode

登录ocdata17,执行

./hdfs namenode -bootstrapStandby

在ocdata16上，ctrl+c 停止namenode
停止journalnode

./stop-dfs-journalnode.sh

5. 日常启动停止¶

启动Zookeeper

分别在ocdata16\ocdata17\ocdata18上执行：

./zkServer.sh start

判断是否执行成功：

echo ruok | nc ocdata16 2183
echo ruok | nc ocdata17 2183
echo ruok | nc ocdata18 2183

如果都返回imok,则代表启动成功

启动HDFS

cd sbin
./start-dfs.sh

停止HDFS

cd sbin
./stop-dfs.sh

6. Yarn

mapred-site.xm

cp ../etc/hadoop/mapred-site.xml.template ../etc/hadoop/mapred-site.xm

<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

yarn-site.xml

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
<value>ocnosql76:8032</value>
</property>
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
<value>ocnosql76:8030</value>
</property>
<property>
<description>The address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
<value>ocnosql76:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>ocnosql76:8031</value>
</property>
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
<value>ocnosql76:8033</value>
</property>

Yarn的启动停止

cd sbin
./start-yarn.sh
./stop-yarn.sh

7. 其他

在安装前，最好在每台机器上确认hostname -i 返回该机器的IP地址
在安装前，确保每个机器上hosts里包含其他机器的主机名和IP的对应关系
如果节点一直在备节点，而不是自动切换到主节点，请重新格式化zkfc：./hdfs zkfc -formatZK
注意：主机名不能含有下划线