nginx错误日志入库小程序

nginx错误日志入库小程序

#!/usr/bin/env python
# -*- coding:utf-8 -*-
# __author__ = 'xiaojinsong'
# __time__ = '2018/2/27'

from __future__ import print_function
import pyinotify
import re
import os
import sys
import MySQLdb
import socket

class ProcessTransientFile(pyinotify.ProcessEvent):
def process_IN_MODIFY(self, event):
line = file.readline()
# print(line)
if line:
intodb(line)

def nginxLogAnalyzer(line):
# print(line,end='')
m = re.match(r'(\d{4}\/\d{2}\/\d{2}\ \d{2}\:\d{2}\:\d{2}) ([^,]*),\s(.*)', line)

dic1 = {}
v = m.group(3).split(', ')
dict1 = dict([(a.split(': ')[0].strip(), a.split(': ')[1].strip()) for a in v])
if dict1['request']:
x = dict1['request']
dict1.pop('request')
dict1['method'] = x.split(' ')[0].strip('"')
dict1['request'] = x.split(' ')[1].strip('"')
dict1['http_version'] = x.split(' ')[2].strip('"')

dict1['time'] = m.group(1)
dict1['errortype'] = m.group(2)
hn = socket.gethostname()
dict1['nginxIP'] = socket.gethostbyname(hn)
return dict1

def intodb(line):
dict1 = nginxLogAnalyzer(line)
field = []
cur.execute("describe nginx_error")
field = cur.fetchall()
mylist = []
for fie in field:
mylist.append(fie[0])
for key in dict1.keys():
if key not in mylist:
try:
sql = "alter table nginx_error add column %s varchar(30) DEFAULT NULL" % key
cur.execute(sql)
conn.commit()
except Exception, e:
conn.rollback()
print(e)
placeholders = ', '.join(['%s'] * len(dict1))
columns = ', '.join(dict1.keys())
sql = "INSERT INTO nginx_error (%s) VALUES (%s)" % (columns, placeholders)
try:
cur.execute(sql, dict1.values())
conn.commit()
except Exception, e:
conn.rollback()
print(e)

if __name__ == '__main__':
conn = MySQLdb.connect(host='xxx.xxx.xxxx', port=3306, user='nginx', passwd='nginx', db='nginx',
charset="utf8")
# cur = conn.cursor(MySQLdb.cursors.DictCursor)
cur = conn.cursor()
cur.execute("SET NAMES utf8")

filename = sys.argv[1]
file = open(filename, 'r')
st_results = os.stat(filename)
st_size = st_results[6]
file.seek(st_size)

wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm)
wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
notifier.loop()