第2周 实施Hadoop集群

Hadoop配置有关文件

完全分布式模式的安装和配置

##配置hosts文件 （所有节点都要做）
[root@hadoop1 hadoop-0.20.2]# cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
192.168.136.128 hadoop1
192.168.136.129 hadoop2
192.168.136.130 hadoop3

##建立hadoop运行账号
这里是测试实验，我就不建了

##配置ssh免密码连入 （所有节点都要做）
[root@hadoop1 ~]# cd
[root@hadoop1 ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
bd:1c:c1:28:f9:9c:06:3d:96:6e:2a:90:c4:6e:22:52 root@hadoop1
[root@hadoop1 ~]# cd /root/.ssh/
[root@hadoop1 .ssh]# cp id_rsa.pub authorized_keys

##分发ssh公钥（所有节点都要做）
##把各个节点的authorized_keys的内容互相拷贝加入到对方的此文件中，然后就可以免密码彼此ssh连入
[root@hadoop1 .ssh]# cat authorized_keys
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5ZMU09PIj6h2/W/PjesZBCRT+D8gzJwbmilswqi3Sde3qXRQz57bqqpexNl5iRbSisQI26OzANpmyg4NZZsSrbxR7Shm8mtb2ytxVa4+r5wlKYfgZe9ssmz1AB6SPRWX64VLWl/lwiJD8Ro94c0QmCHtNJG3Kd4xDUvgIRqzCutWbJVD04oi7qeHtimPlxN3tkEM+Lv1p5azKmn4LR5Nq4e6B371RT6ym+l2E5ByIiXF1DBh94hEk4IrgoNfOvrdDcVymny8bCKc5LM+jGMzE4xRaeC9kRuquKTWtCMIOpEMK6mblMi5eQprjgFo98wBN4EDkeyK9Jtn5mYlgqgH0w==
root@hadoop1
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAshAin3mE3Xj6poC4sYsPC+IIn+z4I1WjiNJn154rVx03qL6oiH4718LDwdKZ0JUrkjsXJ9UrSYEcW3ph0UNNXLzVlCiXyMM9y5KCNh62eeIMmuJxjfjpyY6FAx7LYUb46TSbMUabx+tcGWz6XxU7OafCzbYL5oN0OryaYbxwPPw09h/skO0O+maEr5LFj6QbQQ1Fxw4QrifXmi0K+CizzQtrnDdn4pFvHBOy3oHMgjt+G7qXuIwYbBjPlXdpK/eRY8xYczSNeh94lV39OqPnjx7R5MAc73DcDJPimRqZU//So3nnfY9JaaQkrElMrnQpiNy7iks/rN+CY8pqTr2X4w==
root@hadoop2
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq4Iv5BjmmlVLxcdgW8Ik2dvlKR6gekAkd0fsyFTutV95gjo17cf5gmbOjwQq5tE1AhlFxIa5wmL3e96w6n9kTJcZuh57Pz8mCmMteug4tfxF+iTLgx9S3wzUQ8sMC8H57Vm+tHsC333xLa/yNF4bXGy4G8FXONJg2goo4KTo/D1UVg4MWOCrh5NCtu3uw1f+ekIex+rIED55EWMgVvYni3I4x7QfujN5ztJDLpleVjIdmT50ef/Q/2pKC4prfhpjvqTnPiXvA0Ycmm2IXuvK2FCFQqMswgoJAxvKLwV6DV9/1KTi1BV3XRpthY+dR3wV6Ccj05B/poaoLs/B5sXnZQ==
root@hadoop3

测试ssh（所有节点都要做）
# ssh hadoop1 date
# ssh hadoop2 date
# ssh hadoop3 date

##下载并解压hadoop安装包

hadoop官网：http://hadoop.apache.org/
老版本下载地址：http://archive.apache.org/dist/hadoop/core/hadoop-0.20.2/

##安装jdk（所有节点都要做）
# scp jdk-7u9-linux-i586.tar.gz root@hadoop2:/nosql/hadoop/
# scp jdk-7u9-linux-i586.tar.gz root@hadoop3:/nosql/hadoop/
# tar zxf jdk-7u9-linux-i586.tar.gz
# mv ./jdk1.7.0_09/ /usr/
[root@linux cassandra]# cd
[root@linux ~]# vi .bash_profile

JAVA_HOME=/usr/jdk1.7.0_09

CLASSPATH=.:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH
# java -version
# java
# javac

##下载hadoop压缩包并解压（只在master节点做）
# tar zxf hadoop-0.20.2.tar.gz
[root@hadoop1 hadoop]# cd hadoop-0.20.2/conf/
[root@hadoop1 conf]# ls
capacity-scheduler.xml core-site.xml hadoop-metrics.properties hdfs-site.xml mapred-site.xml slaves ssl-server.xml.example
configuration.xsl hadoop-env.sh hadoop-policy.xml log4j.properties masters ssl-client.xml.example

##配置hadoop-env.sh（只在master节点做）
# The java implementation to use. Required.
# export JAVA_HOME=/usr/lib/j2sdk1.5-sun
export JAVA_HOME=/usr/jdk1.7.0_09

##修改core-site.xml文件（只在master节点做）
##fs.default.name NameNode的IP地址和端口

##特别注意：hadoop的相关配置文件尽量使用主机名而不是ip地址.（使用ip时，执行mapred可能出现“java.lang.IllegalArgumentException: Wrong FS: hdfs://192.168.9.138:9000/user/root/out, expected: hdfs://hadoopm:9000”这样的错误）。

<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://hadoop1:9000</value>
</property>
</configuration>

##修改hdfs-site.xml文件（只在master节点做）
##dfs.data.dir 存放数据文件的目录
##dfs.replication 数据冗余份数
<property>
<name>dfs.data.dir</name>
<value>/nosql/hadoop/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>

##修改mapred-site.xml文件（只在master节点做）
##mapred.job.tracker jobtracker的RPC服务器运行的主机名和端口
<property>
<name>mapred.job.tracker</name>
<value>hadoop1:9001</value>
</property>

##配置masters和slaves文件（只在master节点做）
[root@hadoop1 conf]# cat masters
hadoop1
[root@hadoop1 conf]# cat slaves
hadoop2
hadoop3

##向各节点复制hadoop
# scp -r ./hadoop/ hadoop2:/nosql/
# scp -r ./hadoop/ hadoop3:/nosql/

##格式化namenode（只在master节点做）
[root@hadoop1 nosql]# cd /nosql/hadoop/hadoop-0.20.2/bin/
[root@hadoop1 bin]# ./hadoop namenode -format
14/01/09 05:39:48 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG: host = hadoop1/192.168.136.128
STARTUP_MSG: args = [-format]
STARTUP_MSG: version = 0.20.2
STARTUP_MSG: build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20 -r 911707; compiled by 'chrisdo' on Fri Feb 19 08:07:34 UTC 2010
************************************************************/
14/01/09 05:39:50 INFO namenode.FSNamesystem: fsOwner=root,root,bin,daemon,sys,adm,disk,wheel
14/01/09 05:39:50 INFO namenode.FSNamesystem: supergroup=supergroup
14/01/09 05:39:50 INFO namenode.FSNamesystem: isPermissionEnabled=true
14/01/09 05:39:50 INFO common.Storage: Image file of size 94 saved in 0 seconds.
14/01/09 05:39:50 INFO common.Storage: Storage directory /tmp/hadoop-root/dfs/name has been successfully formatted.
14/01/09 05:39:50 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at hadoop1/192.168.136.128
************************************************************/

##启动hadoop（只在master节点做）
[root@hadoop1 bin]# ./start-all.sh
starting namenode, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-namenode-hadoop1.out
hadoop2: starting datanode, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-datanode-hadoop2.out
hadoop3: starting datanode, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-datanode-hadoop3.out
hadoop1: starting secondarynamenode, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-secondarynamenode-hadoop1.out
starting jobtracker, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-jobtracker-hadoop1.out
hadoop3: starting tasktracker, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-tasktracker-hadoop3.out
hadoop2: starting tasktracker, logging to /nosql/hadoop/hadoop-0.20.2/bin/../logs/hadoop-root-tasktracker-hadoop2.out

##用jps检验各后台进程是否成功启动
[root@hadoop1 bin]# hostname
hadoop1
[root@hadoop1 bin]# jps
4048 NameNode
4374 Jps
4173 SecondaryNameNode
4229 JobTracker
[root@hadoop1 bin]#

[root@hadoop2 hadoop-0.20.2]# hostname
hadoop2
[root@hadoop2 hadoop-0.20.2]# jps
4130 Jps
3965 DataNode
4027 TaskTracker
[root@hadoop2 hadoop-0.20.2]#

[root@hadoop3 nosql]# hostname
hadoop3
[root@hadoop3 nosql]# jps
3953 DataNode
4121 Jps
4015 TaskTracker
[root@hadoop3 nosql]#