python将日志导入数据库代码案例 3

#!/usr/bin/env python
#-*-coding:utf-8-*-
#平台收入日报
#****************************************使用说明****************************************************
# 内网测试通过命令行参数
# -d 指定统计时间
# -p 指定日志所在目录
# -t 指定临时文件存放目录
# 示例如下:
# [ying@localhost tongji]$ ./channelincomeDaily.py -d 20141112 -p /home/zhaoying/log/ -t /home/zhaoying/tongji/
# 外网无需加参数,使用默认值
#****************************************使用说明****************************************************

import MySQLdb
import os, sys, re,string
import time, tarfile,getopt

optmap = {
'dbuser' : 'haoren',
'dbpass' : '123654',
'dbhost' : '192.168.10.13',
'dbport' : 3306,
'dbname' : 'GHKMDB'
}

def get_files(dir, pattern):
res_file_list =[]
if os.path.exists(dir):
cur_file_list = os.listdir(dir)
cur_file_list.sort()
for file_name in cur_file_list:
if re.search(pattern, file_name):
res_file_list.append(file_name)
return res_file_list
else:
return 'no'

def main():

#外网环境默认参数
cur_day = time.strftime("%Y%m%d", time.localtime(time.time()-86400))
log_day = time.strftime('%y%m%d', time.localtime(time.mktime(time.strptime(cur_day, '%Y%m%d'))))
logdirname = "/home/haoren/logdir/%s_67"%log_day
tmpdirname = "/tmp/"
logname = "billserver"

#内网测试指定参数
opts, args = getopt.getopt(sys.argv[1:], 'd:p:t:')
for op, value in opts:
if op == '-d':
m = re.search('[0-9]{8}', value)
if m:
cur_day = value
print cur_day
log_suffix = time.strftime('%y%m%d', time.localtime(time.mktime(time.strptime(cur_day, '%Y%m%d'))))
logname = "billserver.log"
log_day = time.strftime('%y%m%d', time.localtime(time.mktime(time.strptime(cur_day, '%Y%m%d'))))
logdirname = "/home/haoren/logdir/%s_67"%log_day
else:
print "请输入8位日期(比如：20130215)"
return 'no'
elif op == '-p':
logdirname = str(value)
elif op == '-t':
tmpdirname = str(value)

fmt_day = time.strftime('%Y-%m-%d', time.localtime(time.mktime(time.strptime(cur_day, '%Y%m%d'))))
d_time = time.mktime(time.strptime(fmt_day,'%Y-%m-%d'))
print '统计日期:',fmt_day
print '日志名称:',logname
print '日志路径:',logdirname
print '临时文本:',tmpdirname

#平台收入
if os.path.exists("%schannelincome.txt"%tmpdirname):
os.system("rm -f %schannelincome.txt"%tmpdirname)

#平台人民币总量
if os.path.exists("%schannelallgold.txt"%tmpdirname):
os.system("rm -f %schannelallgold.txt"%tmpdirname)

file_list=get_files(logdirname, logname)
for file in file_list:
command = "cat %s/%s | awk '/物品统计/'>> %schannelincome.txt"%(logdirname,file,tmpdirname)
os.system(command)
command = "cat %s/%s | awk '/平台人民币统计/'>> %schannelallgold.txt"%(logdirname,file,tmpdirname)
os.system(command)

#平台收入
filename='%schannelincome.txt'%tmpdirname
a_file = open(filename, 'r')
#平台消耗用户总人民币
allcoin={}
#平台人民币收入
goldincome={}
#平台总人民币
channelallgold = {}
#平台物品
channelprops = {}
#130815-15:01:06 Bill[990] INFO: [物品统计]平台(210043)等级(9)用户(21000162)赠送(1)个物品(3)给歌唱家(21000264),歌唱家等级(1),签约(0), 消耗人民币(10), 歌唱家获得人民币(470), 平台获得人民币(30)
for a_line in a_file.readlines():
m = re.search("^(\S+) Bill\[\d+\] INFO: \[物品统计\]平台\((\d+)\)等级\((\d+)\)用户\((\d+)\)赠送\((\d+)\)个物品\((\d+)\)给歌唱家\((\d+)\),歌唱家等级\((\d+)\),签约\((\d+)\), 消耗人民币\((\d+)\), 歌唱家获得人民币\((\d+)\), 平台获得人民币\((\d+)\)", a_line)
if m:
channelid = int(m.group(2))
propnum = int(m.group(5))
propid = int(m.group(6))
signflag = int(m.group(9))
coin = int(m.group(10))
gold = int(m.group(12))
#离线用户signflag传入的是平台号
if (signflag != 0 and signflag != 1):
if (signflag == int(m.group(2))):
signflag = 1
else:
signflag = 0
if (channelid in channelprops):
if (propid in channelprops[channelid]):
channelprops[channelid][propid] += propnum
allcoin[channelid][propid] += coin
goldincome[channelid][propid][signflag] += gold
else:
channelprops[channelid][propid] = propnum
allcoin[channelid][propid] = coin
goldincome[channelid][propid] = {}
goldincome[channelid][propid][0] = 0
goldincome[channelid][propid][1] = 0
goldincome[channelid][propid][signflag] = gold

else:
channelprops[channelid]={}
channelprops[channelid][propid] = propnum
allcoin[channelid] = {}
allcoin[channelid][propid] = coin
goldincome[channelid]={}
goldincome[channelid][propid]={}
goldincome[channelid][propid][0]=0
goldincome[channelid][propid][1]=0
goldincome[channelid][propid][signflag]=int(m.group(12))

for key in channelprops:
print '平台:',key

a_file.close()

#平台总人民币统计

#平台总人民币
filename='%schannelallgold.txt'%tmpdirname
a_file = open(filename, 'r')
#平台总人民币
allgold={}
#130816-15:28:45 Bill[990] INFO: [平台人民币统计]平台(210043)当前人民币(308864653)
for a_line in a_file.readlines():
m = re.search("^(\S+) Bill\[\d+\] INFO: \[平台人民币统计\]平台\((\d+)\)当前人民币\((\d+)\)", a_line)
if m:
allgold[int(m.group(2))] = int(m.group(3))

for key in allgold:
print'平台:%lu,人民币总量:%u'%(key,allgold[key])

a_file.close()

db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
tabletime = time.strftime("%Y%m%d", time.localtime(time.mktime(time.strptime(cur_day, "%Y%m%d"))))

sql="CREATE TABLE IF NOT EXISTS `CHANNELEARNINGDAILY_%s` like CHANNELEARNINGDAILY_SAMPLE"%tabletime

db_conn.query(sql)
db_conn.query("delete from CHANNELEARNINGDAILY_%s"%tabletime)
for key in channelprops:
if(key not in allgold):
allgold[key]=0
for kitem in channelprops[key]:
db_conn.query("insert into CHANNELEARNINGDAILY_%s(CHANNELID,RECVITEMTOTALCOIN,SIGNGOLD,NONSIGNGOLD,CURGOLD,ITEMID,ITEMNUM ) values(%d,%d,%d,%d,%d,%d,%d)"%(tabletime,int(key),int(allcoin[key][kitem]),int(goldincome[key][kitem][1]),int(goldincome[key][kitem][0]),int(allgold[key]),int(kitem),int(channelprops[key][kitem])))

db_conn.commit()
db_cursor.close()
db_conn.close()
main()
#if __name__ == "__main__":
# main()