数据开发面试准备

1、datanode、namenode、resourceManager，NodeManger的区别和流程

http://www.aboutyun.com/thread-7778-1-1.html

2、hadoop map-reducer的流程，combine的作用

http://blog.csdn.net/lisonglisonglisong/article/details/47125319

3、基本的hadoop wordCount的编写

http://blog.jobbole.com/82607/

http://blog.csdn.net/Jerome_s/article/details/26441151

package com.test.mapreduce.web;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {

public static class WordMapper extends
Mapper<Object, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1);
private Text word = new Text();

public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer itr = new StringTokenizer(line);
while (itr.hasMoreTokens()) {
word.set(itr.nextToken().toLowerCase());
context.write(word, one);
}
}
}

public static class WordReducer extends
Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, new IntWritable(sum));
}
}

public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args)
.getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
Job job = new Job(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(WordMapper.class);
job.setCombinerClass(WordReducer.class);
job.setReducerClass(WordReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}

---------------------------------------------------------------------------------
package cn.dataguru.hadoop;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.TreeMap;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

//位置数据
//IMSI|IMEI|UPDATETYPE|CGI|TIME
//上网数据
//IMSI|IMEI|CGI|TIME|CGI|URL

/**
* 汇总基站数据表
* 计算每个用户在不同的时间段不同的基站停留的时长
* 输入参数 < input path > < output path > < date > < timepoint >
* 参数示例： “/base /output 2012-09-12 09-17-24"
* 意味着以“/base”为输入，"/output"为输出，指定计算2012年09月12日的数据，并分为00-07，07-17，17-24三个时段
* 输出格式 “IMSI|CGI|TIMFLAG|STAY_TIME”
*/
public class BaseStationDataPreprocess extends Configured implements Tool
{
/**
* 计数器
* 用于计数各种异常数据
*/
enum Counter
{
TIMESKIP,       //时间格式有误
OUTOFTIMESKIP,  //时间不在参数指定的时间段内
LINESKIP,       //源文件行有误
USERSKIP        //某个用户某个时间段被整个放弃
}

/**
* 读取一行数据
* 以“IMSI+时间段”作为 KEY 发射出去
*/
public static class Map extends Mapper<LongWritable, Text, Text, Text>
{
String date;
String [] timepoint;
boolean dataSource;

/**
* 初始化
*/
public void setup ( Context context ) throws IOException
{
this.date = context.getConfiguration().get("date");                         //读取日期
this.timepoint = context.getConfiguration().get("timepoint").split("-");    //读取时间分割点

//提取文件名
FileSplit fs = (FileSplit)context.getInputSplit();
String fileName = fs.getPath().getName();
if( fileName.startsWith("POS") )
dataSource = true;
else if ( fileName.startsWith("NET") )
dataSource = false;
else
throw new IOException("File Name should starts with POS or NET");
}

/**
* MAP任务
* 读取基站数据
* 找出数据所对应时间段
* 以IMSI和时间段作为 KEY
* CGI和时间作为 VALUE
*/
public void map ( LongWritable key, Text value, Context context ) throws IOException, InterruptedException
{
String line = value.toString();
TableLine tableLine = new TableLine();

//读取行
try
{
tableLine.set(line, this.dataSource, this.date, this.timepoint );
}
catch ( LineException e )
{
if(e.getFlag()==-1)
context.getCounter(Counter.OUTOFTIMESKIP).increment(1);
else
context.getCounter(Counter.TIMESKIP).increment(1);
return;
}
catch (Exception e)
{
context.getCounter(Counter.LINESKIP).increment(1);
return;
}

context.write( tableLine.outKey(), tableLine.outValue() );
}
}

/**
* 统计同一个IMSI在同一时间段
* 在不同CGI停留的时长
*/
public static class Reduce extends Reducer<Text, Text, NullWritable, Text>
{
private String date;
private SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

/**
* 初始化
*/
public void setup ( Context context )
{
this.date = context.getConfiguration().get("date");                         //读取日期
}

public void reduce ( Text key, Iterable<Text> values, Context context ) throws IOException, InterruptedException
{
String imsi = key.toString().split("\\|")[0];
String timeFlag = key.toString().split("\\|")[1];

//用一个TreeMap记录时间
TreeMap<Long, String> uploads = new TreeMap<Long, String>();
String valueString;

for ( Text value : values )
{
valueString = value.toString();
try
{
uploads.put( Long.valueOf( valueString.split("\\|")[1] ), valueString.split("\\|")[0] );
}
catch ( NumberFormatException e )
{
context.getCounter(Counter.TIMESKIP).increment(1);
continue;
}
}

try
{
//在最后添加“OFF”位置
Date tmp = this.formatter.parse( this.date + " " + timeFlag.split("-")[1] + ":00:00" );
uploads.put ( ( tmp.getTime() / 1000L ), "OFF");

//汇总数据
HashMap<String, Float> locs = getStayTime(uploads);

//输出
for( Entry<String, Float> entry : locs.entrySet() )
{
StringBuilder builder = new StringBuilder();
builder.append(imsi).append("|");
builder.append(entry.getKey()).append("|");
builder.append(timeFlag).append("|");
builder.append(entry.getValue());

context.write( NullWritable.get(), new Text(builder.toString()) );
}
}
catch ( Exception e )
{
context.getCounter(Counter.USERSKIP).increment(1);
return;
}
}

/**
* 获得位置停留信息
*/
private HashMap<String, Float> getStayTime(TreeMap<Long, String> uploads)
{
Entry<Long, String> upload, nextUpload;
HashMap<String, Float> locs = new HashMap<String, Float>();
//初始化
Iterator<Entry<Long, String>> it = uploads.entrySet().iterator();
upload = it.next();
//计算
while( it.hasNext() )
{
nextUpload = it.next();
float diff = (float) (nextUpload.getKey()-upload.getKey()) / 60.0f;
if( diff <= 60.0 )                                  //时间间隔过大则代表关机
{
if( locs.containsKey( upload.getValue() ) )
locs.put( upload.getValue(), locs.get(upload.getValue())+diff );
else
locs.put( upload.getValue(), diff );
}
upload = nextUpload;
}
return locs;
}
}

public int run(String[] args) throws Exception {
Configuration conf = getConf();

conf.set("date", args[2]);
conf.set("timepoint", args[3]);

Job job = new Job(conf, "BaseStationDataPreprocess");
job.setJarByClass(BaseStationDataPreprocess.class);

FileInputFormat.addInputPath( job, new Path(args[0]) );         //输入路径
FileOutputFormat.setOutputPath( job, new Path(args[1]) );       //输出路径

job.setMapperClass( Map.class );                                //调用上面Map类作为Map任务代码
job.setReducerClass ( Reduce.class );                           //调用上面Reduce类作为Reduce任务代码
job.setOutputFormatClass( TextOutputFormat.class );
job.setOutputKeyClass( Text.class );
job.setOutputValueClass( Text.class );

job.waitForCompletion(true);

return job.isSuccessful() ? 0 : 1;
}

public static void main(String[] args) throws Exception
{
if ( args.length != 4 )
{
System.err.println("");
System.err.println("Usage: BaseStationDataPreprocess < input path > < output path > < date > < timepoint >");
System.err.println("Example: BaseStationDataPreprocess /user/james/Base /user/james/Output 2012-09-12 07-09-17-24");
System.err.println("Warning: Timepoints should be begined with a 0+ two digit number and the last timepoint should be 24");
System.err.println("Counter:");
System.err.println("\t"+"TIMESKIP"+"\t"+"Lines which contain wrong date format");
System.err.println("\t"+"OUTOFTIMESKIP"+"\t"+"Lines which contain times that out of range");
System.err.println("\t"+"LINESKIP"+"\t"+"Lines which are invalid");
System.err.println("\t"+"USERSKIP"+"\t"+"Users in some time are invalid");
System.exit(-1);
}

//运行任务
int res = ToolRunner.run(new Configuration(), new BaseStationDataPreprocess(), args);

System.exit(res);
}

}

4、面试题

http://blog.csdn.net/qq_26442553/article/details/78718796