根据web服务器log进行简单的流量分析

最近做的一个网站上线，要做流量统计。网上有一些现成的将js添加到页面上的统计工具，但从后台客户注册数据看，有些没有统计到。做网站的时候比较仓促，自己并未在服务器端进行IP跟踪和计数，幸好服务器提供商默认log了访问者，就依次做做分析。

主要运用了以下几点知识。

1.ftp登录并获取文件；

2.读写文件；

3.对log文件进行分析；

4.容器；

5.IP查询。

自我感觉有三个地方如果能改良就更好了：

1.增加图形界面，产品化。

2.IP地址与实际地址的对应关系，未知哪里有比较权威的参考？

3.log文件的分析，这个demo里只是从log的内容进行了简单的归纳分类，并未从服务器提供商处获取log的规律。

附上代码：IPAnalyze.java

/* @(#)IPAnalyze.java 2010-7-13上午11:27:21
*
* Copyright 2003-2010 TONECON Hearing Equipment co,ltd.All right reserved.
*/

package com.tonecon;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.io.Reader;
import java.net.URL;
import java.net.URLConnection;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import sun.net.TelnetInputStream;
import sun.net.ftp.FtpClient;

/**
*
* @author
* @create-time 2010-7-13上午11:27:21
* @revision 3.0
*/

public class IPAnalyze
{

private static final String ORIGINAL_PATTERN = "dd/MMM/yyyy:HH:mm:ss Z";
private static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
private static final String DATE_PATTERN = "yyyy-MM-dd";

/**
* @param args
* @throws IOException
* @throws ParseException
*/
public static void main(String[] args) throws IOException, ParseException
{
IPAnalyze ip = new IPAnalyze();

// String[] names = new String[2];
// names[0] = "2010-07-12";
// names[1] = "2010-07-13";
// File[] files = ip.getFtpFiles(names);
// Map<String, Set<String>> ipMap = ip.readLogToMap(files);
// ip.outPutReport(ipMap);

String name = "2010-07-14";
File files = ip.getFtpFile(name);
Map<String, Set<String>> ipMap = ip.readLogToMap(files);
ip.outPutReport(ipMap);
}

public File getFtpFile(String name) throws IOException
{
// 连接ftp服务器并读取log文件
FtpClient fc = new FtpClient("ftp address", 21);
fc.login("username", "password");
fc.cd("tomcatlog");
TelnetInputStream tis = fc.get("tomcat_access_log." + name + ".txt");

// 写log文件到本地
File fGetted = new File("D://tomcat_access_log." + name + ".txt");
RandomAccessFile fGet = new RandomAccessFile(fGetted, "rw");
fGet.seek(0);
DataInputStream puts = new DataInputStream(tis);
int ch;
while ((ch = puts.read()) >= 0)
{
fGet.write(ch);
}
puts.close();
fGet.close();
tis.close();
fc.closeServer();
return fGetted;
}

public File[] getFtpFiles(String[] names) throws IOException
{
// 连接ftp服务器并读取log文件
FtpClient fc = new FtpClient("121.199.124.114", 21);
fc.login("zky1f1", "10nian7yueftp");
fc.cd("tomcatlog");
File[] files = new File[names.length];
TelnetInputStream tis = null;
File fGetted = null;
RandomAccessFile fGet = null;
DataInputStream puts = null;
for (int i = 0; i < names.length; i++)
{
tis = fc.get("tomcat_access_log." + names[i] + ".txt");
// 写log文件到本地
fGetted = new File("D://tomcat_access_log." + names[i] + ".txt");
fGet = new RandomAccessFile(fGetted, "rw");
fGet.seek(0);
puts = new DataInputStream(tis);
int ch;
while ((ch = puts.read()) >= 0)
{
fGet.write(ch);
}
puts.close();
fGet.close();
tis.close();
files[i] = fGetted;
}
fc.closeServer();
return files;
}

public Map<String, Set<String>> readLogToMap(File fGetted)
throws IOException, ParseException
{
SimpleDateFormat sdf1 = new SimpleDateFormat(ORIGINAL_PATTERN,
Locale.US);
SimpleDateFormat sdf2 = new SimpleDateFormat(DATETIME_PATTERN);
// 取log文件中有用信息到内存
Map<String, Set<String>> ipMap = new TreeMap<String, Set<String>>();
Reader is = new FileReader(fGetted);
BufferedReader br = new BufferedReader(is);
String s1 = null;
while ((s1 = br.readLine()) != null)
{
String[] segs = s1.split(" - - ");
if (segs[1].contains(".jsp"))
{
String dStr = segs[1].substring(1, 27);
String ip = segs[0];
if (ipMap.containsKey(ip))
{
ipMap.get(ip).add(sdf2.format(sdf1.parse(dStr)));
}
else
{
Set<String> dateSet = new TreeSet<String>();
dateSet.add(sdf2.format(sdf1.parse(dStr)));
ipMap.put(ip, dateSet);
}
}
}
br.close();
is.close();
return ipMap;
}

/**
*
* @param fsGetted
* @return
* @throws IOException
* @throws ParseException
*/
public Map<String, Set<String>> readLogToMap(File[] fsGetted)
throws IOException, ParseException
{
SimpleDateFormat sdf1 = new SimpleDateFormat(ORIGINAL_PATTERN,
Locale.US);
SimpleDateFormat sdf2 = new SimpleDateFormat(DATETIME_PATTERN);
// 取log文件中有用信息到内存
Map<String, Set<String>> ipMap = new TreeMap<String, Set<String>>();
for (File fGetted : fsGetted)
{
Reader fr = new FileReader(fGetted);
BufferedReader br = new BufferedReader(fr);
String s1 = null;
while ((s1 = br.readLine()) != null)
{
String[] segs = s1.split(" - - ");
if (segs[1].contains(".jsp"))
{
String dStr = segs[1].substring(1, 27);
String ip = segs[0];
if (ipMap.containsKey(ip))
{
ipMap.get(ip).add(sdf2.format(sdf1.parse(dStr)));
}
else
{
Set<String> dateSet = new TreeSet<String>();
dateSet.add(sdf2.format(sdf1.parse(dStr)));
ipMap.put(ip, dateSet);
}
}
}
br.close();
fr.close();
}
return ipMap;
}

public void outPutReport(Map<String, Set<String>> ipMap) throws IOException
{
SimpleDateFormat sdf3 = new SimpleDateFormat(DATE_PATTERN);
// 输出结果报告
FileOutputStream fout = new FileOutputStream("D://ipReport."
+ sdf3.format(new Date()) + ".txt");
PrintStream ps = new PrintStream(fout);
URL realUrl = null;
URLConnection conn = null;
BufferedReader br = null;
for (String str : ipMap.keySet())
{
ps.println("IP地址：" + str);
// 使用第三方IP查询接口
realUrl = new URL("http://www.5luyu.cn/get.aspx?ip=" + str);
// URL realUrl = new URL(
// "http://www.youdao.com/smartresult-xml/search.s?type=ip&q="
// + str);
conn = realUrl.openConnection();
conn.connect();
br = new BufferedReader(new InputStreamReader(
conn.getInputStream(), "gbk"));
String line;
ps.print("所在地方：");
while ((line = br.readLine()) != null)
{
ps.println(line);
}

ps.println("于以下时间访问本站：");
Set<String> dateSet = ipMap.get(str);
for (String date : dateSet)
{
ps.println(date);
}
ps.println("----------------------------------------------");

}
ps.println("共计:" + ipMap.size());
fout.close();
}
}