Hadoop2.6 全分布式安装,配置了联邦

Hadoop全分布式安装2.6 共4台机器

hadoop01 namenode zookeeper zkfc(zookeeper failover controller)

hadoop02 namenode datanode zookeeper journalnode zkfc

hadoop03 datanode zookeeper journalnode

hadoop04 datanode journalnode

修改主机名称 永久生效 重启

vi /etc/sysconfig/network

HOSTNAME=hadoop01

HOSTNAME=hadoop02

HOSTNAME=hadoop03

HOSTNAME=hadoop04

临时生效

hostname hadoop01

设置主机名称

vi /etc/hosts

192.168.192.133 hadoop01

192.168.192.134 hadoop02

192.168.192.135 hadoop03

192.168.192.136 hadoop04

scp /etc/hosts root@hadoop02:/etc

scp /etc/hosts root@hadoop03:/etc

scp /etc/hosts root@hadoop04:/etc

ssh登陆其他机器

ssh 192.168.192.133 -l root

输入密码

123456

配置ssh无密码登陆

cd ~

ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa 生成公钥、私钥对

cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys 导入公钥

ssh localhost 测试

登录其它机器 ssh hadoop01 -l root

cd ~

ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa 生成公钥、私钥对

然后用scp命令，把公钥文件发放给hadoop01

scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_02.pub

scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_03.pub

scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_04.pub

登陆hadoop01 导入公钥

cat ~/.ssh/id_dsa_02.pub >> ~/.ssh/authorized_keys

cat ~/.ssh/id_dsa_03.pub >> ~/.ssh/authorized_keys

cat ~/.ssh/id_dsa_04.pub >> ~/.ssh/authorized_keys

将hadoop01上的“最全”公钥，复制到其它机器

scp ~/.ssh/authorized_keys root@hadoop02:/root/.ssh/authorized_keys

scp ~/.ssh/authorized_keys root@hadoop03:/root/.ssh/authorized_keys

scp ~/.ssh/authorized_keys root@hadoop04:/root/.ssh/authorized_keys

安装JDK 略过

上传并解压hadoop2.6

tar -zxvf hadoop-2.6.0-x64.tar.gz -C /opt/modules/

cd /opt/modules/hadoop-2.6.0/etc/hadoop

vi hadoop-env.sh

export JAVA_HOME=/opt/modules/jdk1.7

vi yarn-env.sh

export JAVA_HOME=/opt/modules/jdk1.7

vi mapred-env.sh

export JAVA_HOME=/opt/modules/jdk1.7

vi core-site.xml tmp目录如不存在，则先mkdir手动创建

<configuration>

<property>

<name>hadoop.tmp.dir</name>

<value>/opt/hadoop/tmp</value>

</property>

<property>

<name>fs.defaultFS</name>

<value>hdfs://mycluster</value>

</property>

<property>

<name>ha.zookeeper.quorum</name>

<value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>

</property>

</configuration>

rm -rf /opt/hadoop/

mkdir /opt/hadoop

mkdir /opt/hadoop/tmp

mkdir /opt/hadoop/journal

vi hdfs-site.xml dfs.namenode.name.dir和dfs.namenode.edits.dir所设置的目录都应该手工建立好

配置联邦两个namenode

<property>

<name>dfs.nameservices</name>

<value>mycluster</value>

</property>

<property>

<name>dfs.ha.namenodes.mycluster</name>

<value>nn1,nn2</value>

</property>

<property>

<name>dfs.namenode.rpc-address.mycluster.nn1</name>

<value>hadoop01:8020</value>

</property>

<property>

<name>dfs.namenode.rpc-address.mycluster.nn2</name>

<value>hadoop02:8020</value>

</property>

<property>

<name>dfs.namenode.http-address.mycluster.nn1</name>

<value>hadoop01:50070</value>

</property>

<property>

<name>dfs.namenode.http-address.mycluster.nn2</name>

<value>hadoop02:50070</value>

</property>

<property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://hadoop02:8485;hadoop03:8485;hadoop04:8485/mycluster</value>

</property>

<property>

<name>dfs.client.failover.proxy.provider.mycluster</name>

<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

</property>

<property>

<name>dfs.ha.fencing.methods</name>

<value>sshfence</value>

</property>

<property>

<name>dfs.ha.fencing.ssh.private-key-files</name>

<value>/root/.ssh/id_dsa</value>

</property>

<property>

<name>dfs.journalnode.edits.dir</name>

<value>/opt/hadoop/journal</value>

</property>

<property>

<name>dfs.ha.automatic-failover.enabled</name>

<value>true</value>

</property>

<property>

<name>dfs.replication</name>

<value>2</value>

</property>

cp /opt/modules/hadoop-2.6.0/etc/hadoop/mapred-site.xml.template mapred-site.xml

vi mapred-site.xml

<property>

<name>mapreduce.framework.name</name>

<value>yarn</value>

</property>

vi yarn-site.xml

<property>

<name>yarn.nodemanager.aux-services</name>

<value>mapreduce_shuffle</value>

</property>

vi slaves

hadoop02

hadoop03

hadoop04

安装zookeeper

tar -zxvf zookeeper-3.4.6.tar.gz -C /opt/modules/

cd /opt/modules/zookeeper-3.4.6/conf

cp zoo_sample.cfg zoo.cfg

vi zoo.cfg

mkdir /opt/zookeeper

tickTime=2000

initLimit=10

syncLimit=5

dataDir=/opt/zookeeper

server.1=hadoop01:2888:3888

server.2=hadoop02:2888:3888

server.3=hadoop03:2888:3888

cd /opt/zookeeper 在hadoop01操作

vi myid

1

ssh hadoop02 -l root

scp -r /opt/zookeeper root@hadoop02:/opt/

vi myid

2

ssh hadoop03 -l root

scp -r /opt/zookeeper root@hadoop03:/opt/

vi myid

3

scp -r /opt/zookeeper root@hadoop03:/opt/

scp -r /opt/modules/zookeeper-3.4.6 root@hadoop02:/opt/modules

scp -r /opt/modules/zookeeper-3.4.6 root@hadoop03:/opt/modules

vi /etc/profile

export ZK_HOME=/opt/modules/zookeeper-3.4.6

export PATH=$PATH:$ZK_HOME/bin

source /etc/profile

scp /etc/profile root@hadoop02:/etc/

scp /etc/profile root@hadoop03:/etc/

启动zookeeper 01-02-03

zkServer.sh start

停止ZooKeeper服务

zkServer.sh stop

jps 查看是否启动

2516 QuorumPeerMain

拷贝hadoop

scp -r /opt/modules/hadoop-2.6.0 root@hadoop02:/opt/modules

scp -r /opt/modules/hadoop-2.6.0 root@hadoop03:/opt/modules

scp -r /opt/modules/hadoop-2.6.0 root@hadoop04:/opt/modules

02-03-04分别创建目录

mkdir /opt/hadoop

mkdir /opt/hadoop/tmp

mkdir /opt/hadoop/journal

mkdir /opt/hadoop/dfs

mkdir /opt/hadoop/dfs/data

mkdir /opt/hadoop/dfs/name

启动Hadoop

cd /opt/modules/hadoop-2.6.0/sbin/

02-03-04启动journalnode

./hadoop-daemon.sh start journalnode

格式化 zkfc(zookeeper failover controller)

01

cd /opt/modules/hadoop-2.6.0/bin/

./hdfs zkfc -formatZK

格式化namenode

01

cd /opt/modules/hadoop-2.6.0/bin/

./hdfs namenode -format

启动一个namenode

cd /opt/modules/hadoop-2.6.0/sbin/

./hadoop-daemon.sh start namenode

在没有格式化的namenode上执行

cd /opt/modules/hadoop-2.6.0/bin/

./hdfs namenode -bootstrapStandby

停止服务(zookeeper除外)

./stop-dfs.sh

全部启动

./start-dfs.sh

测试访问

hadoop01:50070

hadoop02:50070

启动yarn

./start-yarn.sh

查看日志前50行

tail -n50 /opt/modules/hadoop-2.6.0/logs/hadoop-root- .log

配置hadoop环境变量

vi /etc/profile

##HADOOP

export HADOOP_HOME=/opt/modules/hadoop-2.6.0

export PATH=$PATH:/opt/modules/hadoop-2.6.0/sbin:/opt/modules/hadoop-2.6.0/bin

source /etc/profile

scp /etc/profile root@hadoop02:/etc/

scp /etc/profile root@hadoop03:/etc/

scp /etc/profile root@hadoop04:/etc/

拷贝hadoop配置

scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop02:/opt/modules/hadoop-2.6.0

scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop03:/opt/modules/hadoop-2.6.0

scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop04:/opt/modules/hadoop-2.6.0

每次启动01-02-03

zkServer.sh start

全部启动

./start-dfs.sh

启动yarn

./start-yarn.sh

停止服务(zookeeper除外)

./stop-dfs.sh

./stop-yarn.sh

批量关机小脚本

#!/bin/bash

ip_array=("hadoop02" "hadoop03" "hadoop04")

for ip in ${ip_array[*]}

do

ssh root@$ip "shutdown -h now"

done

echo "shutdown02 03 04"

ssh root@hadoop01 "shutdown -h now"

chmod +x ./test.sh