Hadoop 学习笔记 （十一） MapReduce 求平均成绩

china：
张三78
李四89
王五96
赵六67
english
张三80
李四82
王五84
赵六86
math
张三88
李四99
王五66
赵六77

importjava.io.IOException;

importorg.apache.hadoop.conf.Configuration;
importorg.apache.hadoop.fs.Path;
importorg.apache.hadoop.io.IntWritable;
importorg.apache.hadoop.io.FloatWritable;
importorg.apache.hadoop.io.Text;
importorg.apache.hadoop.mapreduce.Job;
importorg.apache.hadoop.mapreduce.Mapper;
importorg.apache.hadoop.mapreduce.Reducer;
importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;
importorg.apache.hadoop.mapreduce.lib.input.TextInputFormat;
importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
importorg.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;

publicclassMyAverage{

publicstaticclassFormatMapperextendsMapper<Object,Text,Text,IntWritable>{

privateIntWritableval=newIntWritable();

publicvoidmap(Objectkey,Textvalue,Contextcontext)throwsIOException,InterruptedException{
Stringline[]=value.toString().split("\\s");
val.set(Integer.parseInt(line[1]));
context.write(newText(line[0]),val);
}
}

publicstaticclassAverageReducerextendsReducer<Text,IntWritable,Text,FloatWritable>{
publicvoidreduce(Textkey,Iterable<IntWritable>values,Contextcontext)throwsIOException,InterruptedException{
intcount=0;
intsum=0;
for(IntWritableval:values){
sum+=val.get();
count++;
}
floatave=(float)sum/count;
context.write(key,newFloatWritable(ave));
}
}

publicstaticvoidmain(String[]args)throwsException{
Stringdir_in="hdfs://localhost:9000/in_average";
Stringdir_out="hdfs://localhost:9000/out_average";

Pathin=newPath(dir_in);
Pathout=newPath(dir_out);

Configurationconf=newConfiguration();
JobaverageJob=newJob(conf,"my_average");

averageJob.setJarByClass(MyAverage.class);

averageJob.setInputFormatClass(TextInputFormat.class);
averageJob.setMapperClass(FormatMapper.class);
averageJob.setCombinerClass(IntSumReducer.class);
//countJob.setPartitionerClass(HashPartitioner.class);
averageJob.setMapOutputKeyClass(Text.class);
averageJob.setMapOutputValueClass(IntWritable.class);

FileInputFormat.addInputPath(averageJob,in);

averageJob.setReducerClass(AverageReducer.class);
//averageJob.setNumReduceTasks(1);
averageJob.setOutputKeyClass(Text.class);
averageJob.setOutputValueClass(FloatWritable.class);
//countJob.setOutputFormatClass(SequenceFileOutputFormat.class);

FileOutputFormat.setOutputPath(averageJob,out);

averageJob.waitForCompletion(true);

}

}

张三82.0
李四90.0
王五82.0
赵六76.666664