036 关于网站的UV分析

一:准备

1.统计的维度

　　guid

　　tracktime

　　provice

2.key与value的设定

　　key：date+provice_guid

　　value：NullWritable

3.案例分析

　　表示某天某个省份的某个人无论访问网站多少次，仅仅记做一次访问统计

　　UV：统计页面访问的总人数---》userID对于用户进行去重

二：程序

1.map程序

　　


　　

2.reduce程序

　　


3.结果

　　


4.理解点

　　1）怎么去重

　　　　数据key的形式：date+provice_guid。

　　　　当guid是相同的时候，在shuffle的group分组时，key被分组，一起的放在一起，而value则是nullwritable，没有使用value。

　　　　所以到达reduce的时候，数据已经被去重了。

　　2）NullWritable.get()

　　　　使用反射，获得NullWritable的对象。

5.完整程序

1 package com.senior.network;
2
3 import java.io.IOException;
4 import java.util.HashMap;
5 import java.util.Map;
6 import java.util.Set;
7
8 import org.apache.commons.lang.StringUtils;
9 import org.apache.hadoop.conf.Configuration;
10 import org.apache.hadoop.conf.Configured;
11 import org.apache.hadoop.fs.Path;
12 import org.apache.hadoop.io.IntWritable;
13 import org.apache.hadoop.io.LongWritable;
14 import org.apache.hadoop.io.NullWritable;
15 import org.apache.hadoop.io.Text;
16 import org.apache.hadoop.mapreduce.Job;
17 import org.apache.hadoop.mapreduce.Mapper;
18 import org.apache.hadoop.mapreduce.Mapper.Context;
19 import org.apache.hadoop.mapreduce.Reducer;
20 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
21 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
22 import org.apache.hadoop.util.Tool;
23 import org.apache.hadoop.util.ToolRunner;
24
25 public class WebUvCount extends Configured implements Tool{
26     //Mapper
27     public static class WebUvCountMapper extends Mapper<LongWritable,Text,Text,NullWritable>{
28         private Text mapoutputkey=new Text();
29         @Override
30         protected void cleanup(Context context) throws IOException,InterruptedException {
31
32         }
33         @Override
34         protected void setup(Context context) throws IOException,InterruptedException {
35
36         }
37
38         @Override
39         protected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {
40             String lineValue=value.toString();
41             String[] strs=lineValue.split("\t");
42             if(30>strs.length){
43                 context.getCounter("webPvMapper_counter", "length_LT_30").increment(1L);
44                 return;
45             }
46             String guidValue=strs[5];    //
47             if(StringUtils.isEmpty(guidValue)){
48                 return;
49             }
50             String trackTimeValue=strs[17];
51             if(StringUtils.isEmpty(trackTimeValue)){
52                 return;
53             }
54             String dateVAlue=trackTimeValue.substring(0,13);//
55             String priviceIdValue=strs[23];
56
57             Integer priviceId=Integer.MAX_VALUE;
58             try{
59                 priviceId = Integer.valueOf(priviceIdValue);  //
60             }catch(Exception e){
61                 return;
62             }
63
64             mapoutputkey.set(dateVAlue+"\t"+priviceIdValue+"_"+guidValue);
65             context.write(mapoutputkey,NullWritable.get());
66         }
67
68     }
69
70
71
72     //Reducer
73     public static class WebUvCountReducer extends Reducer<Text,NullWritable,Text,IntWritable>{
74         private Text outputkey=new Text();
75         private Map<String,Integer> dateMap;
76         private IntWritable outputvalue=new IntWritable();
77
78         @Override
79         protected void setup(Context context)throws IOException, InterruptedException {
80             dateMap=new HashMap<String,Integer>();
81         }
82
83         @Override
84         protected void reduce(Text key, Iterable<NullWritable> values,Context context)throws IOException, InterruptedException {
85             String date=key.toString().split("_")[0];
86             if(dateMap.containsKey(date)){
87                 Integer previousUV=dateMap.get(date);
88                 Integer uv=previousUV+1;
89                 dateMap.put(date, uv);
90             }else{
91                 dateMap.put(date, 1);
92             }
93         }
94
95         @Override
96         protected void cleanup(Context context)throws IOException, InterruptedException {
97             Set<String> dateSet=dateMap.keySet();
98             for(String date:dateSet){
99                 Integer uv=dateMap.get(date);
100                 outputkey.set(date);
101                 outputvalue.set(uv);
102                 context.write(outputkey, outputvalue);
103             }
104         }
105
106
107     }
108
109     //Driver
110     public int run(String[] args)throws Exception{
111         Configuration conf=this.getConf();
112         Job job=Job.getInstance(conf,this.getClass().getSimpleName());
113         job.setJarByClass(WebUvCount.class);
114         //input
115         Path inpath=new Path(args[0]);
116         FileInputFormat.addInputPath(job, inpath);
117
118         //output
119         Path outpath=new Path(args[1]);
120         FileOutputFormat.setOutputPath(job, outpath);
121
122         //map
123         job.setMapperClass(WebUvCountMapper.class);
124         job.setMapOutputKeyClass(Text.class);
125         job.setMapOutputValueClass(NullWritable.class);
126
127         //shuffle
128
129         //reduce
130         job.setReducerClass(WebUvCountReducer.class);
131         job.setOutputKeyClass(Text.class);
132         job.setOutputValueClass(IntWritable.class);
133
134         //submit
135         boolean isSucess=job.waitForCompletion(true);
136         return isSucess?0:1;
137     }
138
139     //main
140     public static void main(String[] args)throws Exception{
141         Configuration conf=new Configuration();
142         //compress
143         conf.set("mapreduce.map.output.compress", "true");
144         conf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec");
145         args=new String[]{
146                 "hdfs://linux-hadoop01.ibeifeng.com:8020/user/beifeng/mapreduce/wordcount/inputWebData",
147                 "hdfs://linux-hadoop01.ibeifeng.com:8020/user/beifeng/mapreduce/wordcount/outputWebData6"
148         };
149         int status=ToolRunner.run(new WebUvCount(), args);
150         System.exit(status);
151     }
152
153 }