Hadoop中利用自身类实现json格式转换为对象。

假如在大数据中我们遇到了下面JSON格式的数据，我们想统计每部电影rate的综合。比较正规的做法是通过json解析成为对象的格式。虽然我们可以通过JSON-Simple导入jar包的方式处理，但实际上最方便的还是利用

hadoop自身
org.codehaus.jackson.map.ObjectMapper;类直接处理最为简单

{"movie":"1193","rate":"5","timeStamp":"978300760","uid":"1"}
{"movie":"661","rate":"3","timeStamp":"978302109","uid":"1"}
{"movie":"914","rate":"3","timeStamp":"978301968","uid":"1"}
{"movie":"3408","rate":"4","timeStamp":"978300275","uid":"1"}
{"movie":"2355","rate":"5","timeStamp":"978824291","uid":"1"}
{"movie":"1197","rate":"3","timeStamp":"978302268","uid":"1"}

首先你需要定义一个javabean用来存储数据，这个bean需要实现Writable接口，因为网络直接传输对象是需要序列化的。

不了解序列化的同学，可以复习一下java序列化的知识。

package com.mre.moviecount;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Writable;

public class MovieBean implements Writable{
private long movie;
private long rate;
private long timeStamp;
private long uid;

public MovieBean(){}
public MovieBean(long movie,long rate){
this.movie=movie;
this.rate = rate;
}
public long getMovie() {
return movie;
}
public void setMovie(long movie) {
this.movie = movie;
}
public long getRate() {
return rate;
}
public void setRate(long rate) {
this.rate = rate;
}
public long gettimeStamp() {
return timeStamp;
}
public void settimeStamp(long timeStamp) {
this.timeStamp = timeStamp;
}
public long getUid() {
return uid;
}
public void setUid(long uid) {
this.uid = uid;
}

@Override
public void write(DataOutput out) throws IOException {
//把对象序列化的过程
out.writeLong(movie);
out.writeLong(rate);
out.writeLong(timeStamp);
out.writeLong(uid);

}
@Override
public void readFields(DataInput in) throws IOException {
//把把序列化的对象读取到内存中
movie = in.readLong();
rate =in.readLong();
timeStamp =in.readLong();
uid=in.readLong();
}

@Override
public String toString() {
return "movie：" + movie + ", rate=" + rate;
}

}


然后就是正常的流程mapper和reducer来实现具体的机能。
package com.mre.moviecount;

import java.io.IOException;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;

public class RateCount {
public static class MyMapper extends Mapper<Object, Text, LongWritable, MovieBean> {

protected void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String line = value.toString();
ObjectMapper om = new ObjectMapper();//把读取的json转化为对象
MovieBean moviebean = om.readValue(line, MovieBean.class);
//把movie对象和和id放入map处理 ※可以直接放rate，
context.write(new LongWritable(moviebean.getMovie()), moviebean);
}

}

public static class MyReducer extends Reducer<LongWritable, MovieBean, LongWritable, MovieBean> {

@Override
protected void reduce(LongWritable key, Iterable<MovieBean> values, Context context)
throws IOException, InterruptedException {

long count =0;
for(MovieBean mb:values){
count+=mb.getRate();//把movie为单位的rate累加
}

//我们把count和movie存入对象，最终把对象输出到hdfs的文件中
//输出的格式就是我们在MovieBean中的toString方法定义的格式。
MovieBean moviebean = new MovieBean(key.get(),count);
context.write(key, moviebean);
}

}

public static void main(String[] args) throws Exception{
Job job = Job.getInstance();
job.setJarByClass(RateCount.class);
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);

// job.setMapOutputKeyClass(Text.class);
// job.setMapOutputValueClass(NullWritable.class);

job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(MovieBean.class);

FileInputFormat.addInputPath(job, new Path("hdfs://192.168.32.30:9000/input/movie.json" ));

FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.32.30:9000/output6"));
job.waitForCompletion(true);
}
}

最后输出的结果

379 movie：379, rate=1260
380 movie：380, rate=5088
381 movie：381, rate=688
382 movie：382, rate=798
383 movie：383, rate=882
384 movie：384, rate=116
385 movie：385, rate=34
386 movie：386, rate=78
387 movie：387, rate=280
388 movie：388, rate=353
389 movie：389, rate=74
390 movie：390, rate=209
391 movie：391, rate=131
392 movie：392, rate=70
393 movie：393, rate=195
394 movie：394, rate=66
396 movie：396, rate=4