Python基于nginx访问日志并统计IP访问量

如果想看看Nginx有多少IP访问量，有哪些国家访问，并显示IP地址的归属地分布，python可以结合使用高春辉老师ipip.net 【 免费版 IP 地址数据库 】，Shell可以使用nali，我这边主要使用python语言来实现需求,并将查询结果以邮件形式发送，也是为了学习和回顾python语言。很感谢高春辉老师提供的免费版IP地址数据库。一、Ningx日志如下：

41.42.97.104 - - [26/Feb/2015:03:35:40 -0500] "GET /root/ HTTP/1.1" 301 20 "http://baibai.123.com/09" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" - 0.562
41.42.97.104 - - [26/Feb/2015:03:35:41 -0500] "GET /crossadkla.xml HTTP/1.1" 304 0 "https://baibai.123.com/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" - 0.000
99.122.189.203 - - [26/Feb/2015:03:35:42 -0500] "GET /root/ HTTP/1.1" 301 20 "http://baibai.123.com/11" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" - 0.562
99.122.189.203  - - [26/Feb/2015:03:35:44 -0500] "GET /crossadkla.xml HTTP/1.1" 304 0 "https://baibai.123.com/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" - 0.000
99.122.189.203  - - [26/Feb/2015:03:35:44 -0500] "GET /crossadkla.xml HTTP/1.1" 304 0 "https://baibai.123.com/" "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.115 Safari/537.36" - 0.000

二、下载 免费版 IP 地址数据库

#wget  http://s.qdcdn.com/17mon/17monipdb.zip #unzip  17monipdb.zip

三、IP库常见问题FAQ示例代码：

import osfrom ipip import IPfrom ipip import IPX

IP.load(os.path.abspath("mydata4vipday2.dat"))print IP.find("118.28.8.8")

IPX.load(os.path.abspath("mydata4vipday2.datx"))print IPX.find("118.28.8.8")

执行输出：

中国  天津  天津      鹏博士
中国  天津  天津      鹏博士   39.128399   117.185112  Asia/Shanghai   UTC+8   120000

IP库guihub地址： https://github.com/17mon/python四、Python 统计代码

#encoding=utf8import re,sys,os,csv,smtplibfrom ipip import IPfrom ipip import IPXfrom email import encodersfrom email.mime.multipart import MIMEMultipartfrom email.mime.base import MIMEBasefrom email.mime.text import MIMETextfrom optparse import OptionParser
reload(sys)
sys.setdefaultencoding('utf-8')print sys.getdefaultencoding()nginx_log_path="/app/nginx/logs/apptest_www.access.log"pattern = re.compile(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}')def stat_ip_views(log_path):  ret={}  f = open(log_path, "r")  for line in f:    match = pattern.match(line)    if match:      ip=match.group(0)      if ip in ret:        views=ret[ip]      else:        views=0      views=views+1      ret[ip]=views  return retdef run():  ip_views=stat_ip_views(nginx_log_path)  max_ip_view={}  fileName='out.csv'  f=open('out.csv','w+')  b = 'IP,国家,访问数总数'  print >> f,b  for ip in ip_views:    IP.load(os.path.abspath("17monipdb.dat"))    count=IP.find("%s"% (ip))    conut_s=count.split()    countery=conut_s[0]    views=ip_views[ip]    c = '%s,%s,%s' %(ip,countery,views)    print >> f,c    if len(max_ip_view)==0:      max_ip_view[ip]=views    else:      _ip=max_ip_view.keys()[0]      _views=max_ip_view[_ip]      if views>_views:        max_ip_view[ip]=views        max_ip_view.pop(_ip)    print "IP:", ip, "国家:", countery, "访问数:", views
print "总共有多少IP:", len(ip_views)  print "最大访问IP数:", max_ip_view  g = ""  d = '总共有多少IP:%s' %(len(ip_views))  e = '最大访问IP数:%s' %(max_ip_view)  print >> f,g  print >> f,d  print >> f,edef sendMail(html,emailaddress,mailSubject,from_address="other@test.com"):    mail_list=emailaddress.split(",")    msg=MIMEMultipart()    msg['Accept-Language']='zh-CN'    msg['Accept-Charset']= 'ISO-8859-1,utf-8'    msg['From']=from_address    msg['to']=";".join(mail_list)    msg['Subject']=mailSubject.decode("utf-8")    txt=MIMEText(html,'html','utf-8')    txt.set_charset('utf-8')    msg.attach(txt)    file=MIMEBase('application', 'octet-stream')    file.set_payload(open(fileName, 'rb').read())    encoders.encode_base64(file)    file.add_header('Content-Disposition', 'attachment; filename="%s"' % os.path.basename(fileName))    msg.attach(file)    smtp=smtplib.SMTP("mail.test.com")    smtp.sendmail(msg["From"],mail_list,msg.as_string())    smtp.close()if __name__ == '__main__':  run()  fileName='out.csv'  cmd = 'iconv -f UTF8 -t GB18030 %s -o %s.bak && mv %s.bak %s' %(fileName,fileName,fileName,fileName)  os.system(cmd)  Content= 'Dear ALL: <br>	附件内国家IP访问数据分析统计，请查收！  <br>	如有任何问题，请及时与我联系！'  Subject = '[分析]国家创建数据IP分析统计'  sendMail(html=Content,emailaddress='kuangl@test.com',mailSubject=Subject)

五、执行结果

utf-8
IP: 41.42.97.104 国家: 埃及 访问数: 2
IP: 99.122.189.203 国家: 美国 访问数: 3
总共有多少IP: 2
最大访问IP数: {'99.122.189.203': 3}

六、邮件发送结果

本文来自：Linux教程网