MapReduce入门级之WordCount单词计数

话不多说直接贴上代码：具体的实现代码后面描述

package com.whomai.test;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount{

public static class mapWordCount extends Mapper<Object,Text,Text,IntWritable>{
private  final static IntWritable one = new IntWritable(1);
private Text word =  new Text();
public void map(Object key,Text value,Context context) throws IOException, InterruptedException{
String line =value.toString();
StringTokenizer st = new StringTokenizer(line);
while(st.hasMoreTokens()){
word.set(st.nextToken());
context.write(word, one);
}
}
}
public static class reduceWordCount extends Reducer<Text,IntWritable,Text,IntWritable>{
public void reduce(Text key,Iterable<IntWritable> values,Context context) throws IOException, InterruptedException{
int sum = 0;
for(IntWritable val : values){
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{
Configuration conf = new Configuration();
Job job = new Job(conf,"word");
job.setJarByClass(WordCount.class);
String wordCountInput = "hdfs://192.168.248.133:9000/wordCountInput";
String WordCountOut="hdfs://192.168.248.133:9000/WordCountOutPath";
job.setMapperClass(mapWordCount.class);
job.setCombinerClass(reduceWordCount.class);
job.setReducerClass(reduceWordCount.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(wordCountInput));
FileOutputFormat.setOutputPath(job, new Path(WordCountOut));
System.exit(job.waitForCompletion(true) ? 0 : 1);

}
}

首先hadoop是基于分布式文件系统hdfs和分布式处理MapReduce的。如字面，Mapreduce包含了Map操作和Reduce操作。

Map操作就是拆分，将大量繁重的任务分散给各个节点。那么reduce就是将map的任务进行整合。

在代码中我们实现了map和reduce的内部类。Map类继承了hadoop2.0中的mapper类。里面传递了四个参数，分别是传入文件时的key，value类型和，map输出时的类型。

这样来说，

mapreduce的基本思想就是这些，各个程序相辅相成，比如说去重，其主要涵义就是在在reduce的过程中将value的值设置为1就行了。