用Python将统计数据不存在的记录按维度对应指标补齐(续：日数据情形)

接【Python多线程跑Hive批一例】-->http://blog.csdn.net/babyfish13/article/details/54906405

及【用Python将统计数据不存在的记录按维度对应指标补齐】-->http://blog.csdn.net/babyfish13/article/details/54632508

本文主要是将数据按日存储的结果表有些维度为空的记录给补起来，与按周的又有所不同。
1、用到的日期临时表
show create table bitmp_date_list;

CREATE TABLE `bitmp_date_list` (
`str_date` varchar(10) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

2、改进及更新后的日期处理脚本
/Users/nisj/PycharmProjects/EsDataProc/RemainProcByDay/DayProc.py

# -*- coding=utf-8 -*-
import warnings
import datetime

warnings.filterwarnings("ignore")

def getNowDay():
DayNow = datetime.datetime.today().strftime('%Y-%m-%d')
return DayNow

def getFristDay():
FristDay=datetime.datetime.strptime('2015-06-29', '%Y-%m-%d').strftime('%Y-%m-%d')
return FristDay

def getDayAllList(run_day):
DayAllList=[]
begin_date = datetime.datetime.strptime(getFristDay(), "%Y-%m-%d")

if run_day != '':
end_date = datetime.datetime.strptime(run_day, '%Y-%m-%d')
else:
end_date = datetime.datetime.strptime(getNowDay(), "%Y-%m-%d")

while begin_date <= end_date:
date_str = begin_date.strftime("%Y-%m-%d")
DayAllList.append(date_str)
begin_date += datetime.timedelta(days=1)
return DayAllList

def getDayRunList(run_day):
DayRunList=[]
if run_day != '':
end_date = datetime.datetime.strptime(run_day, '%Y-%m-%d')
else:
end_date = datetime.datetime.strptime(getNowDay(), "%Y-%m-%d")

if end_date - datetime.timedelta(days=60) <= datetime.datetime.strptime(getFristDay(), "%Y-%m-%d"):
begin_date = datetime.datetime.strptime(getFristDay(), "%Y-%m-%d")
else:
begin_date = end_date - datetime.timedelta(days=60)

while begin_date <= end_date:
date_str = begin_date.strftime("%Y-%m-%d")
DayRunList.append(date_str)
begin_date += datetime.timedelta(days=1)

return DayRunList

def getDayRunListReverse(run_day):
DayRunList=[]
begin_date = datetime.datetime.strptime(run_day, '%Y-%m-%d') + datetime.timedelta(days=1)

if begin_date + datetime.timedelta(days=60) >= datetime.datetime.strptime(getNowDay(), "%Y-%m-%d") - datetime.timedelta(days=1):
end_date = datetime.datetime.strptime(getNowDay(), "%Y-%m-%d") - datetime.timedelta(days=1)
else:
end_date = begin_date + datetime.timedelta(days=60)

while begin_date <= end_date:
date_str = begin_date.strftime("%Y-%m-%d")
DayRunList.append(date_str)
begin_date += datetime.timedelta(days=1)

return DayRunList

def getDayRemainRunIntervalList(str_begin_date, str_end_date):
DayList=[]
begin_date = datetime.datetime.strptime(str_begin_date, "%Y-%m-%d")
end_date = datetime.datetime.strptime(str_end_date, "%Y-%m-%d")

while begin_date <= end_date:
date_str = begin_date.strftime("%Y-%m-%d")
DayList.append(date_str)
begin_date += datetime.timedelta(days=1)
return DayList

# Batch Test
# run_day = '2016-07-10'
# for remain_day in getDayAllList(run_day):
#     for his_day in getDayRunList(run_day=remain_day):
#         print remain_day, his_day

# one day
# for his_day in getDayRunList(run_day=run_day):
#     print run_day, his_day

# run_day = '2017-01-01'
# for remain_day in getDayRunList(run_day):
#     print remain_day,run_day
# for his_day in getDayRunList(run_day=remain_day):
#     print remain_day, his_day

# 计算2017年的数据信息
# str_begin_date = '2017-01-01'
# str_end_date = '2017-02-05'
# for remain_day in getDayRemainRunIntervalList(str_begin_date, str_end_date):
#     for his_day in getDayRunList(run_day=remain_day):
#         print remain_day,his_day

3、空记录查找及插入
/Users/nisj/PycharmProjects/EsDataProc/RemainProcByDay/Daily_result_null_proc.py

# -*- coding=utf-8 -*-
import os
import re
from DayProc import *

def Insert_TmpDate_Data(run_day):
os.system("""mysql -hMysqlHost -PMysqlPort -uMysqlUser -pMysqlPass -N -e "use funnyai_data; truncate table bitmp_date_list; " """ )
for date_str in getDayRunListReverse(run_day):
os.system("""mysql -hMysqlHost -PMysqlPort -uMysqlUser -pMysqlPass -N -e "use funnyai_data; \
insert into bitmp_date_list(str_date) \
select '%s'; \
" """ % (date_str))

def resultNullProc(run_day):
result_data = os.popen("""mysql -hMysqlHost -PMysqlPort -uMysqlUser -pMysqlPass -N -e "use funnyai_data; \
select appkey,app_source,date \
from chushou_user_compare_daily_stat \
where date='%s' and app_source <>'' \
group by appkey,app_source,date; \
" """ % (run_day)).readlines();

nrpd_list = []
for nrp_list in result_data:
nrp = re.split('\t', nrp_list.replace('\n', ''))
nrpd_list.append(nrp)
for nrpd in nrpd_list:
appkey = nrpd[0]
appsource = nrpd[1]
date = nrpd[2]

result_data1 = os.popen("""mysql -hMysqlHost -PMysqlPort -uMysqlUser -pMysqlPass -N -e "use funnyai_data; \
select '%s' date,a1.str_date compare_date,'%s' appkey,'%s' app_source \
from bitmp_date_list a1 \
left join ( \
select appkey,app_source,date,compare_date \
from chushou_user_compare_daily_stat \
where date='%s' and appkey='%s' and app_source='%s' \
) a2 on a1.str_date=a2.compare_date \
where a2.appkey is null \
order by a2.date,a1.str_date;" """ % (date, appkey, appsource, date, appkey, appsource)).readlines();
nrpd_list = []
for nrp_list in result_data1:
nrp = re.split('\t', nrp_list.replace('\n', ''))
nrpd_list.append(nrp)
for nrpd in nrpd_list:
date = nrpd[0]
compare_date = nrpd[1]
appkey = nrpd[2]
appsource = nrpd[3]
print date,compare_date,appkey,appsource
os.system("""mysql -hMysqlHost -PMysqlPort -uMysqlUser -pMysqlPass -N -e "use funnyai_data; \
insert into chushou_user_compare_daily_stat(appkey,app_source,date,compare_date,compare_identify_count,compare_register_user_count,compare_user_count,compare_pay_amount,compare_pay_user_count,compare_pay_count) \
select '%s','%s','%s','%s','%d','%d','%d','%d','%d','%d'; \
" """ % (appkey, appsource, date, compare_date, 0, 0, 0, 0, 0, 0))

# run_day = '2017-01-01'
# Insert_TmpDate_Data(run_day)
# resultNullProc(run_day)

for run_day in getDayRemainRunIntervalList('2017-01-02', '2017-01-03'):
print run_day
Insert_TmpDate_Data(run_day)
resultNullProc(run_day)