windows下idea中搭建hadoop开发环境，向远程hadoop集群提交mapreduce任务

1.下载hadoop2.6.0-cdh5.6.1,解压并安装

公司用的hadoop集群版本是hadoop2.6.0-cdh5.6.1，防止版本冲突，所有的hadoop版本号都用了这个。

下载地址：http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.6.1.tar.gz

解压，放在D:\software\hadoop-2.6.0

配置环境变量：

HADOOP_HOME=D:\software\hadoop-2.6.0

HADOOP_BIN_PATH=%HADOOP_HOME%\bin

HADOOP_PREFIX=D:\software\hadoop-2.6.0

在Path环境变量后追加 ;%HADOOP_HOME%\bin

下载winutils.exe 文件，放在D:\software\hadoop-2.6.0\bin目录下，不然运行程序时会报一个错误

2.编写代码

代码是仿造《hadoop权威指南》一书中的第二章的示例代码

pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>

<groupId>com.cyq</groupId>
<artifactId>HadoopDemo</artifactId>
<version>1.0-SNAPSHOT</version>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.0-cdh5.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>2.6.0-cdh5.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0-cdh5.6.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>2.6.0-mr1-cdh5.6.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit-dep</artifactId>
<version>4.8.2</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.6</source>
<target>1.6</target>
</configuration>
</plugin>
</plugins>
</build>

</project>

map任务

package com.cyq;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/**
* @author Administrator
* @date 2017/11/27
**/

public class MapTemp extends Mapper<LongWritable,Text,Text,IntWritable> {

@Override
public void map(LongWritable longWritable, Text text, Context context) {
//        读取文件内容
String line = text.toString();
//读取年份
String year =line.substring(0,4);
int quality=Integer.parseInt(line.substring(5,7));
try {
context.write(new Text(year),new IntWritable(quality));
}catch (Exception e){

}

}
}

reduce任务

package com.cyq;/**
* Created by Administrator on 2017/11/27.
*/

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.Iterator;

/**
* @author Administrator
* @date 2017/11/27
**/

public class Reduce extends Reducer<Text,IntWritable,Text,IntWritable> {

@Override
public void reduce(Text text, Iterable<IntWritable> iterable,Context context){
int maxValue=Integer.MIN_VALUE;
while (iterable.iterator().hasNext()){
maxValue=Math.max(maxValue,iterable.iterator().next().get());
}
try {
context.write(text,new IntWritable(maxValue));
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

main程序

package com.cyq;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
* @author Administrator
* @date 2017/11/27
**/

public class MaxTemperature extends Configured implements Tool{

@Override
public int run(String[] args) throws Exception{

Configuration conf = new Configuration();
conf.addResource("core-site.xml");
conf.addResource("hdfs-site.xml");
conf.addResource("mapred-site.xml");
conf.addResource("yarn-site.xml");
conf.set("mapreduce.job.jar", "D:\\chen_demo\\HadoopDemo\\target\\HadoopDemo-1.0-SNAPSHOT.jar");
conf.set("mapreduce.framework.name", "yarn");
conf.set("yarn.resourcemanager.hostname", "172.16.50.80");
conf.set("mapreduce.app-submission.cross-platform", "true");
Job job = Job.getInstance(conf);
job.setJarByClass(MaxTemperature.class);
job.setJobName("max temperature");
//        设置输入输出路径
FileInputFormat.addInputPath(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));

job.setMapperClass(MapTemp.class);
job.setReducerClass(Reduce.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
System.exit(job.waitForCompletion(true) ? 0:1);

return 1;

}

public static void main(String[] args) throws Exception {
ToolRunner.run(new MaxTemperature(), args);
}
}

3.运行程序

虽然博客写的很简单，但是新手入门，在实际搭建的时候还是遇到很多问题，没有详细写出来，多数是因为版本不一致导致的问题。也跟我一样在搭建该环境的小伙伴们如果遇到问题，希望能在评论区能互相交流一下经验，总结错误。