您的位置：首页 > Web前端

HDFS HA+Federation的部署（多机）-配置文件

2015-09-06 15:04 495 查看

配置文件具体内容：

1、hadoop-env.sh

# Copyright 2011 The Apache Software Foundation

#

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0
#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# Set Hadoop-specific environment variables here.

# The only required environment variable is JAVA_HOME. All others are

# optional. When running a distributed configuration it is best to

# set JAVA_HOME in this file, so that it is correctly defined on

# remote nodes.

# The java implementation to use.

export JAVA_HOME=/usr/lib/jvm/java-6-sun

# The jsvc implementation to use. Jsvc is required to run secure datanodes.

#export JSVC_HOME=${JSVC_HOME}

export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}

# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler.

for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do

if [ "$HADOOP_CLASSPATH" ]; then

export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f

else

export HADOOP_CLASSPATH=$f

fi

done

# The maximum amount of heap to use, in MB. Default is 1000.

#export HADOOP_HEAPSIZE=

#export HADOOP_NAMENODE_INIT_HEAPSIZE=""

# Extra Java runtime options. Empty by default.

export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"

# Command specific options appended to HADOOP_OPTS when specified

export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"

export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"

export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"

# The following applies to multiple commands (fs, dfs, fsck, distcp etc)

export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"

#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS"

# On secure datanodes, user to run the datanode as after dropping privileges

export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}

# Where log files are stored. $HADOOP_HOME/logs by default.

#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER

# Where log files are stored in the secure data environment.

export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}

# The directory where pid files are stored. /tmp by default.

# NOTE: this should be set to a directory that can only be written to by

# the user that will run the hadoop daemons. Otherwise there is the

# potential for a symlink attack.

export HADOOP_PID_DIR=${HADOOP_PID_DIR}

export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}

# A string representing this instance of hadoop. $USER by default.

export HADOOP_IDENT_STRING=$USER

2、mapred-site.xml

<?xml version="1.0"?>



<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>







<configuration>



<property>

<name>mapreduce.framework.name</name>

<value>yarn</value>

<description>The runtime framework for executing MapReduce jobs.

Can be one of local, classic or yarn.

</description>

</property>



<property>

<name>mapreduce.jobhistory.address</name>

<value>SY-0355:10020</value>

<description>MapReduce JobHistory Server IPC host:port</description>

</property>

<property>

<name>mapreduce.jobhistory.webapp.address</name>

<value>SY-0355:19888</value>

<description>MapReduce JobHistory Server Web UI host:port</description>

</property>

</configuration>

3、core-site.xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>









<configuration>

<property>

<name>fs.defaultFS</name>

<value>hdfs://SY-0217:8020</value>

<description>The name of the default file system. A URI whose

scheme and authority determine the FileSystem implementation. The

uri's scheme determines the config property (fs.SCHEME.impl) naming

the FileSystem implementation class. The uri's authority is used to

determine the host, port, etc. for a filesystem.</description>

</property>

</configuration>

4、hdfs-site.xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>









<configuration>

<property>

<name>dfs.nameservices</name>

<value>hadoop-cluster1,hadoop-cluster2</value>

<description>

Comma-separated list of nameservices.

</description>

</property>



<property>

<name>dfs.ha.namenodes.hadoop-cluster1</name>

<value>nn1,nn2</value>

<description>

The prefix for a given nameservice, contains a comma-separated

list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster1.nn1</name>

<value>SY-0217:8020</value>

<description>

RPC address for nomenode1 of hadoop-cluster1

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster1.nn2</name>

<value>SY-0355:8020</value>

<description>

RPC address for nomenode2 of hadoop-test

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster1.nn1</name>

<value>SY-0217:50070</value>

<description>

The address and the base port where the dfs namenode1 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster1.nn2</name>

<value>SY-0355:50070</value>

<description>

The address and the base port where the dfs namenode2 web ui will listen on.

</description>

</property>



<property>

<name>dfs.ha.namenodes.hadoop-cluster2</name>

<value>nn3,nn4</value>

<description>

The prefix for a given nameservice, contains a comma-separated

list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster2.nn3</name>

<value>SY-0226:8020</value>

<description>

RPC address for nomenode1 of hadoop-cluster1

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster2.nn4</name>

<value>SY-0225:8020</value>

<description>

RPC address for nomenode2 of hadoop-test

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster2.nn3</name>

<value>SY-0226:50070</value>

<description>

The address and the base port where the dfs namenode1 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster2.nn4</name>

<value>SY-0225:50070</value>

<description>

The address and the base port where the dfs namenode2 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.name.dir</name>

<value>file:///home/dongxicheng/hadoop/hdfs/name</value>

<description>Determines where on the local filesystem the DFS name node

should store the name table(fsimage). If this is a comma-delimited list

of directories then the name table is replicated in all of the

directories, for redundancy. </description>

</property>

<property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://SY-0355:8485;SY-0225:8485;SY-0226:8485/hadoop-cluster</value>

<description>A directory on shared storage between the multiple namenodes

in an HA cluster. This directory will be written by the active and read

by the standby in order to keep the namespaces synchronized. This directory

does not need to be listed in dfs.namenode.edits.dir above. It should be

left empty in a non-HA cluster.

</description>

</property>

<property>

<name>dfs.datanode.data.dir</name>

<value>file:///home/dongxicheng/hadoop/hdfs/data</value>

<description>Determines where on the local filesystem an DFS data node

should store its blocks. If this is a comma-delimited

list of directories, then data will be stored in all named

directories, typically on different devices.

Directories that do not exist are ignored.

</description>

</property>

<property>

<name>dfs.ha.automatic-failover.enabled</name>

<value>false</value>

<description>

Whether automatic failover is enabled. See the HDFS High

Availability documentation for details on automatic HA

configuration.

</description>

</property>

<property>

<name>dfs.journalnode.edits.dir</name>

<value>/home/dongxicheng/hadoop/hdfs/journal/</value>

</property>

</configuration>

5、yarn-site.xml

<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>









<configuration>

<property>

<name>dfs.nameservices</name>

<value>hadoop-cluster1,hadoop-cluster2</value>

<description>

Comma-separated list of nameservices.

</description>

</property>



<property>

<name>dfs.ha.namenodes.hadoop-cluster1</name>

<value>nn1,nn2</value>

<description>

The prefix for a given nameservice, contains a comma-separated

list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster1.nn1</name>

<value>SY-0217:8020</value>

<description>

RPC address for nomenode1 of hadoop-cluster1

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster1.nn2</name>

<value>SY-0355:8020</value>

<description>

RPC address for nomenode2 of hadoop-test

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster1.nn1</name>

<value>SY-0217:50070</value>

<description>

The address and the base port where the dfs namenode1 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster1.nn2</name>

<value>SY-0355:50070</value>

<description>

The address and the base port where the dfs namenode2 web ui will listen on.

</description>

</property>



<property>

<name>dfs.ha.namenodes.hadoop-cluster2</name>

<value>nn3,nn4</value>

<description>

The prefix for a given nameservice, contains a comma-separated

list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster2.nn3</name>

<value>SY-0226:8020</value>

<description>

RPC address for nomenode1 of hadoop-cluster1

</description>

</property>

<property>

<name>dfs.namenode.rpc-address.hadoop-cluster2.nn4</name>

<value>SY-0225:8020</value>

<description>

RPC address for nomenode2 of hadoop-test

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster2.nn3</name>

<value>SY-0226:50070</value>

<description>

The address and the base port where the dfs namenode1 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.http-address.hadoop-cluster2.nn4</name>

<value>SY-0225:50070</value>

<description>

The address and the base port where the dfs namenode2 web ui will listen on.

</description>

</property>

<property>

<name>dfs.namenode.name.dir</name>

<value>file:///home/dongxicheng/hadoop/hdfs/name</value>

<description>Determines where on the local filesystem the DFS name node

should store the name table(fsimage). If this is a comma-delimited list

of directories then the name table is replicated in all of the

directories, for redundancy. </description>

</property>

<property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://SY-0355:8485;SY-0225:8485;SY-0226:8485/hadoop-cluster1</value>



<description>A directory on shared storage between the multiple namenodes

in an HA cluster. This directory will be written by the active and read

by the standby in order to keep the namespaces synchronized. This directory

does not need to be listed in dfs.namenode.edits.dir above. It should be

left empty in a non-HA cluster.

</description>

</property>

<property>

<name>dfs.datanode.data.dir</name>

<value>file:///home/dongxicheng/hadoop/hdfs/data</value>

<description>Determines where on the local filesystem an DFS data node

should store its blocks. If this is a comma-delimited

list of directories, then data will be stored in all named

directories, typically on different devices.

Directories that do not exist are ignored.

</description>

</property>

<property>

<name>dfs.ha.automatic-failover.enabled</name>

<value>false</value>

<description>

Whether automatic failover is enabled. See the HDFS High

Availability documentation for details on automatic HA

configuration.

</description>

</property>

<property>

<name>dfs.journalnode.edits.dir</name>

<value>/home/dongxicheng/hadoop/hdfs/journal/</value>

</property>

</configuration>

6、fairscheduler.xml

<?xml version="1.0"?>

<allocations>

<queue name="infrastructure">

<minResources>102400 mb, 50 vcores </minResources>

<maxResources>153600 mb, 100 vcores </maxResources>

<maxRunningApps>200</maxRunningApps>

<minSharePreemptionTimeout>300</minSharePreemptionTimeout>

<weight>1.0</weight>

<aclSubmitApps>root,yarn,search,hdfs</aclSubmitApps>

</queue>

<queue name="tool">

<minResources>102400 mb, 30 vcores</minResources>

<maxResources>153600 mb, 50 vcores</maxResources>

</queue>

<queue name="sentiment">

<minResources>102400 mb, 30 vcores</minResources>

<maxResources>153600 mb, 50 vcores</maxResources>

</queue>

</allocations>

7、slaves

SY-0226

SY-0225

SY-0355

注意：所有操作均在Hadoop部署目录下进行。

启动Hadoop集群：

-------------------------------------------------------------------

(1) 启动nn1与nn2

Step1 :

在各个JournalNode节点上，输入以下命令启动journalnode服务：

sbin/hadoop-daemon.sh start journalnode

Step2:

在[nn1]上，对其进行格式化，并启动：

bin/hdfs namenode -format -clusterId hadoop-cluster

sbin/hadoop-daemon.sh start namenode

Step3:

在[nn2]上，同步nn1的元数据信息：

bin/hdfs namenode -bootstrapStandby

Step4:

启动[nn2]：

sbin/hadoop-daemon.sh start namenode

经过以上四步操作，nn1和nn2均处理standby状态

Step5:

将[nn1]切换为Active

bin/hdfs haadmin -ns hadoop-cluster1 -transitionToActive nn1

-------------------------------------------------------------------

(2) 启动nn3与nn4

Step1:

在[nn3]上，对其进行格式化，并启动：

bin/hdfs namenode -format -clusterId hadoop-cluster

sbin/hadoop-daemon.sh start namenode

Step2:

在[nn4]上，同步nn3的元数据信息：

bin/hdfs namenode -bootstrapStandby

Step3:

启动[nn4]：

sbin/hadoop-daemon.sh start namenode

经过以上三步操作，nn3和nn4均处理standby状态

Step4:

将[nn3]切换为Active

bin/hdfs haadmin -ns hadoop-cluster2 -transitionToActive nn3

-------------------------------------------------------------------

（3）启动所有datanode

Step6:

在[nn1]上，启动所有datanode

sbin/hadoop-daemons.sh start datanode

-------------------------------------------------------------------

（4）关闭Hadoop集群：

在[nn1]上，输入以下命令

sbin/stop-dfs.sh

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签：

相关文章推荐

新的分享

章节导航