hadoop学习(3)
2015-06-15 19:10
169 查看
求学生成绩平均值 数据格式 学生名称 成绩
Map函数
Reduce函数
Main函数
Map函数
package com.hadoop.myhadoop1; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Mapper; public class Map extends Mapper<LongWritable,Text,Text,LongWritable> { @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { // TODO Auto-generated method stub String line = value.toString(); System.out.println(line); StringTokenizer tokel = new StringTokenizer(line,"\n"); while (tokel.hasMoreTokens()) { StringTokenizer toke2 = new StringTokenizer(tokel.nextToken()); String strname = toke2.nextToken(); String strscorent = toke2.nextToken(); Text name = new Text(strname); LongWritable scorent = new LongWritable(Integer.parseInt(strscorent)); context.write(name, scorent); } } }
Reduce函数
package com.hadoop.myhadoop1; import java.io.IOException; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Reducer; public class Reduce extends Reducer<Text, LongWritable, Text, LongWritable> { @Override protected void reduce(Text key, Iterable<LongWritable> value, Context context) throws IOException, InterruptedException { // TODO Auto-generated method stub int sum = 0; int count = 0; for (LongWritable scort : value) { sum += scort.get(); count++; } long aver = sum/count; context.write(key, new LongWritable(aver)); } }
Main函数
package com.hadoop.myhadoop1; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; public class Process { public static void main(String[] args) throws IOException, ReflectiveOperationException, InterruptedException { Configuration conf = new Configuration(); conf.set("fs.default.name", "hdfs://192.168.1.108:9000"); String[] ars = new String[]{"input","output"}; String[] otherArgs = new GenericOptionsParser(conf,ars).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage:wodcount"); System.exit(2); } Job job = new Job(conf, "Score_Process"); job.setJarByClass(Process.class); job.setMapperClass(Map.class); job 4000 .setCombinerClass(Reduce.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); FileInputFormat.setInputPaths(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.out.println(job.waitForCompletion(true)? 1 : 0); } }
相关文章推荐
- 详解HDFS Short Circuit Local Reads
- Hadoop_2.1.0 MapReduce序列图
- 使用Hadoop搭建现代电信企业架构
- 单机版搭建Hadoop环境图文教程详解
- hadoop常见错误以及处理方法详解
- hadoop 单机安装配置教程
- hadoop的hdfs文件操作实现上传文件到hdfs
- hadoop实现grep示例分享
- Apache Hadoop版本详解
- linux下搭建hadoop环境步骤分享
- hadoop client与datanode的通信协议分析
- hadoop中一些常用的命令介绍
- Hadoop单机版和全分布式(集群)安装
- 用PHP和Shell写Hadoop的MapReduce程序
- hadoop map-reduce中的文件并发操作
- Hadoop1.2中配置伪分布式的实例
- java结合HADOOP集群文件上传下载
- 用python + hadoop streaming 分布式编程(一) -- 原理介绍,样例程序与本地调试
- Hadoop安装感悟
- hadoop安装lzo