Hadoop之MapReduce调度：通过shell进行多日期的串行跑批统计

#!/usr/bin/env python
# encoding: utf-8
import sys

# 输入为标准输入stdin
for line in sys.stdin:
if '/room/m-1015.htm' in line:
print '%s' % (line)

#!/usr/bin/env bash

rm -rf result.txt
for dataDate in 2017-08-21 2017-08-22 2017-08-23 2017-08-24 2017-08-25 2017-08-26 2017-08-27 2017-08-28;
do
echo $dataDate
hadoop dfs -rm -r -skipTrash /nisj/mp_result;
hadoop jar /opt/apps/hadoop-2.7.2/share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar \
-mapper /home/hadoop/nisj/hitsCalc3/filter_mapperOnly.py -file /home/hadoop/nisj/hitsCalc3/filter_mapperOnly.py \
-input /tmp/oss_access/$dataDate/*_localhost_access_log.$dataDate.*.txt \
-output /nisj/mp_result

#hadoop dfs -cat /nisj/mp_result/*
hitsNum=`hadoop dfs -cat /nisj/mp_result/* |grep -v "^$"|wc -l`
echo $dataDate '--->' $hitsNum >> result.txt
done