用多线程实现的Java爬虫程序

参数：private static int webDepth = 2;//爬虫深度。主页的深度为1，设置深度后超过该深度的网页不会抓取。

private int intThreadNum = 10;//线程数。开启的线程数。

抓取时也会在程序源文件目录下生成一个report.txt文件记录爬虫的运行情况，并在抓取结束后生成一个fileindex.txt文件维护网页文件索引。

本程序用到了多线程(静态变量和同步)，泛型，文件操作，URL类和连接，Hashtable类关联数组，正则表达式及其相关类。运行时需使用命令行参数，第一个参数应使用http://开头的有效URL字符串作为爬虫的主页，第二个参数（可选）应输入可转换为int型的字符串（用Integer.parseInt(String s)静态方法可以转换的字符串，如3）作为爬虫深度，如果没有，则默认深度为2。

本程序的不足之处是：只考虑了href= href=' href="后加绝对url的这三种情况(由于url地址在网页源文件中情况比较复杂，有时处理也会出现错误)，还有相对url和window.open('的情况没有考虑。异常处理程序也只是简单处理。如果读者有改进办法可以把源代码帖出，不胜感激。

附上源代码如下（保存名为GetWeb.java）：



import java.io.File;

import java.io.BufferedReader;

import java.io.FileOutputStream;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStreamWriter;

import java.io.PrintWriter;

import java.net.URL;

import java.net.URLConnection;

import java.util.ArrayList;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import java.util.Hashtable;

public class GetWeb {

private int webDepth = 2;//爬虫深度

private int intThreadNum = 10;//线程数

private String strHomePage = "";//主页地址

private String myDomain;//域名

private String fPath = "web";//储存网页文件的目录名

private ArrayList<String> arrUrls = new ArrayList<String>();//存储未处理URL

private ArrayList<String> arrUrl = new ArrayList<String>();//存储所有URL供建立索引

private Hashtable<String,Integer> allUrls = new Hashtable<String,Integer>();//存储所有URL的网页号

private Hashtable<String,Integer> deepUrls = new Hashtable<String,Integer>();//存储所有URL深度

private int intWebIndex = 0;//网页对应文件下标，从0开始

private String charset = "GB2312";

private String report = "";

private long startTime;

private int webSuccessed = 0;

private int webFailed = 0;





public GetWeb(String s)

{

this.strHomePage = s;

}

public GetWeb(String s,int i)

{

this.strHomePage = s;

this.webDepth = i;

}



public synchronized void addWebSuccessed()

{

webSuccessed++;

}

public synchronized void addWebFailed()

{

webFailed++;

}

public synchronized void addReport(String s)

{

try

{

report += s;

PrintWriter pwReport = new PrintWriter(new FileOutputStream("report.txt"));

pwReport.println(report);

pwReport.close();

}

catch(Exception e)

{

System.out.println("生成报告文件失败!");

}

}

public synchronized String getAUrl()

{

String tmpAUrl = arrUrls.get(0);

arrUrls.remove(0);

return tmpAUrl;

}

public synchronized String getUrl()

{

String tmpUrl = arrUrl.get(0);

arrUrl.remove(0);

return tmpUrl;

}

public synchronized Integer getIntWebIndex()

{

intWebIndex++;

return intWebIndex;

}

/**

* @param args

*/

public static void main(String[] args)

{

if (args.length == 0 || args[0].equals(""))

{

System.out.println("No input!");

System.exit(1);

}

else if(args.length == 1)

{

GetWeb gw = new GetWeb(args[0]);

gw.getWebByHomePage();

}

else

{

GetWeb gw = new GetWeb(args[0],Integer.parseInt(args[1]));

gw.getWebByHomePage();

}

}

public void getWebByHomePage()

{

startTime = System.currentTimeMillis();

this.myDomain = getDomain();

if (myDomain == null)

{

System.out.println("Wrong input!");

//System.exit(1);

return;

}

System.out.println("Homepage = " + strHomePage);

addReport("Homepage = " + strHomePage + "!\n");

System.out.println("Domain = " + myDomain);

addReport("Domain = " + myDomain + "!\n");

arrUrls.add(strHomePage);

arrUrl.add(strHomePage);

allUrls.put(strHomePage,0);

deepUrls.put(strHomePage,1);

File fDir = new File(fPath);

if(!fDir.exists())

{

fDir.mkdir();

}

System.out.println("Start!");

this.addReport("Start!\n");

String tmp = getAUrl();

this.getWebByUrl(tmp,charset,allUrls.get(tmp)+"");

int i = 0;

for (i=0;i<intThreadNum;i++)

{

new Thread(new Processer(this)).start();

}

while (true)

{

if(arrUrls.isEmpty() && Thread.activeCount() == 1)

{

long finishTime = System.currentTimeMillis();

long costTime = finishTime-startTime;

System.out.println("\n\n\n\n\nFinished!");

addReport("\n\n\n\n\nFinished!\n");

System.out.println("Start time = " + startTime + " " + "Finish time = " + finishTime + " " + "Cost time = " + costTime + "ms");

addReport("Start time = " + startTime + " " + "Finish time = " + finishTime + " " + "Cost time = " + costTime + "ms" + "\n");

System.out.println("Total url number = " + (webSuccessed+webFailed) + " Successed: " + webSuccessed + " Failed: " + webFailed);

addReport("Total url number = " + (webSuccessed+webFailed) + " Successed: " + webSuccessed + " Failed: " + webFailed + "\n");



String strIndex = "";

String tmpUrl = "";

while (!arrUrl.isEmpty())

{

tmpUrl = getUrl();

strIndex += "Web depth:" + deepUrls.get(tmpUrl) + " Filepath: " + fPath + "/web" + allUrls.get(tmpUrl) + ".htm" + " url:" + tmpUrl + "\n\n";

}

System.out.println(strIndex);

try

{

PrintWriter pwIndex = new PrintWriter(new FileOutputStream("fileindex.txt"));

pwIndex.println(strIndex);

pwIndex.close();

}

catch(Exception e)

{

System.out.println("生成索引文件失败!");

}

break;

}

}

}

public void getWebByUrl(String strUrl,String charset,String fileIndex)

{

try

{

//if(charset==null||"".equals(charset))charset="utf-8";

System.out.println("Getting web by url: " + strUrl);

addReport("Getting web by url: " + strUrl + "\n");

URL url = new URL(strUrl);

URLConnection conn = url.openConnection();

conn.setDoOutput(true);

InputStream is = null;

is = url.openStream();



String filePath = fPath + "/web" + fileIndex + ".htm";

PrintWriter pw = null;

FileOutputStream fos = new FileOutputStream(filePath);

OutputStreamWriter writer = new OutputStreamWriter(fos);

pw = new PrintWriter(writer);

BufferedReader bReader = new BufferedReader(new InputStreamReader(is));

StringBuffer sb = new StringBuffer();

String rLine = null;

String tmp_rLine = null;

while ( (rLine = bReader.readLine()) != null)

{

tmp_rLine = rLine;

int str_len = tmp_rLine.length();

if (str_len > 0)

{

sb.append("\n" + tmp_rLine);

pw.println(tmp_rLine);

pw.flush();

if (deepUrls.get(strUrl) < webDepth)getUrlByString(tmp_rLine,strUrl);

}

tmp_rLine = null;

}

is.close();

pw.close();

System.out.println("Get web successfully! " + strUrl);

addReport("Get web successfully! " + strUrl + "\n");

addWebSuccessed();

}

catch (Exception e)

{

System.out.println("Get web failed! " + strUrl);

addReport("Get web failed! " + strUrl + "\n");

addWebFailed();

}

}

public String getDomain()

{

String reg = "(?<=http\\://[a-zA-Z0-9]{0,100}[.]{0,1})[^.\\s]*?\\.(com|cn|net|org|biz|info|cc|tv)";

Pattern p = Pattern.compile(reg,Pattern.CASE_INSENSITIVE);

Matcher m = p.matcher(strHomePage);

boolean blnp = m.find();

if (blnp == true)

{

return m.group(0);

}

return null;

}

public void getUrlByString(String inputArgs,String strUrl)

{

String tmpStr = inputArgs;

String regUrl = "(?<=(href=)[\"]?[\']?)[http://][^\\s\"\'\\?]*(" + myDomain + ")[^\\s\"\'>]*";

Pattern p = Pattern.compile(regUrl,Pattern.CASE_INSENSITIVE);

Matcher m = p.matcher(tmpStr);

boolean blnp = m.find();

//int i = 0;

while (blnp == true)

{

if (!allUrls.containsKey(m.group(0)))

{

System.out.println("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0));

addReport("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0) + "\n");

arrUrls.add(m.group(0));

arrUrl.add(m.group(0));

allUrls.put(m.group(0),getIntWebIndex());

deepUrls.put(m.group(0),(deepUrls.get(strUrl)+1));

}

tmpStr = tmpStr.substring(m.end(),tmpStr.length());

m = p.matcher(tmpStr);

blnp = m.find();

}

}



class Processer implements Runnable

{

GetWeb gw;

public Processer(GetWeb g)

{

this.gw = g;

}

public void run()

{

//Thread.sleep(5000);

while (!arrUrls.isEmpty())

{

String tmp = getAUrl();

getWebByUrl(tmp,charset,allUrls.get(tmp)+"");

}

}

}

}