python 日志分析统计脚本

http://www.apparitor.info/2012/03/19/python-%E6%97%A5%E5%BF%97%E5%88%86%E6%9E%90%E7%BB%9F%E8%AE%A1%E8%84%9A%E6%9C%AC/

脚本类似shell的tail功能，用来统计每分钟的4xx、5xx的状态码数量，统计php执行时间小于1秒、1-5秒、5秒以上的数量，和每秒的并发请求。将结果放到/tmp目录下， 也可以用cacti将结果画图。

#!/usr/bin/env python
import time, os

#-----------------------------
log_file = '/var/log/httpd/cmi_access_log'
#-----------------------------

log_name = log_file.split('/')[-1]
file = open(log_file, 'r')
lt1,to1_5,gt5,status4,status5,concurrent = 0,0,0,0,0,0
Time2 = '0'
st_results = os.stat(log_file)
st_size = st_results[6]
file.seek(st_size)
size = os.path.getsize(log_file)

while 1:
Time = time.strftime("%Y_%m_%d",time.localtime(time.time() - 60))
where = file.tell()
line = file.readline()
L = line.split('"')
Time1 = time.strftime("%Y_%m_%d_%H:%M",time.localtime(time.time() - 60))
if not line:
size = os.path.getsize(log_file)
if size < where:
file = open(log_file, 'r')
else:
time.sleep(1)
file.seek(where)
else:
try:
Phptime = float(L[-2])
url = L[1]
status = L[2][1:4]
if '4' in status[0]:
status4 += 1
elif '5' in status[0]:
status5 += 1
if Time1 in Time2:
if 'f5.php' not in url:
concurrent += 1
if Phptime < 1:
lt1 += 1
elif 1 <= Phptime <= 5:
to1_5 += 1
else:
gt5 += 1
else:
concurrent = concurrent / 60
out = Time1 + "," + str(lt1) + "," + str(to1_5) + "," + str(gt5) + "," + str(status4) + "," + str(status5) + "," + str(concurrent)
F = open('/tmp/' + log_name + '_' + Time + '.data', 'a')
print >> F, out
F.close()
Time2 = Time1
lt1,to1_5,gt5,status4,status5,concurrent = 0,0,0,0,0,0
except:
pass