对爬虫技术的理解--Crawler

爬虫的基本思路如下：
根据URl获取相应页面的html代码
利用正则匹配或者Jsoup等库解析html代码，提取需要的内容
将获取的内容持久化到数据库中
处理好中文字符的编码问题，可以采用多线程提高效率
测试通过的demo：
实体bean：
public class CrawlStat{

private long totalLinks;

private int totalProcessedPages;

private long totalTextSize;

public long getTotalLinks() {

return totalLinks;

}

public int getTotalProcessedPages() {

return totalProcessedPages;

}

public long getTotalTextSize() {

return totalTextSize;

}

public void incProcessedPages() {

this.totalProcessedPages++;

}

public void incTotalLinks(int count) {

this.totalLinks += count;

}

public void incTotalTextSize(int count) {

this.totalTextSize += count;

}

public void setTotalLinks(long totalLinks) {

this.totalLinks = totalLinks;

}

public void setTotalProcessedPages(int totalProcessedPages) {

this.totalProcessedPages = totalProcessedPages;

}

public void setTotalTextSize(long totalTextSize) {

this.totalTextSize = totalTextSize;

}

}
具体的是先方法：
public class MyCrawlers extends WebCrawler{

/** 爬取数据保存文件路径 */

private final static String CVS_PATH = "data/crawl/ziroom.csv";

/** 爬取匹配原则 */

private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g|ico"

+ "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

private final static String LINK_PATH = "data/crawl/link.csv";//爬取link文件路径

private static final Logger logger = LoggerFactory.getLogger(MyCrawler.class);

private final static String URL_PREFIX = "http://sh.ziroom.com/z/nl";

private final File csv;

private final File csv2;

private CsvWriter cw;

private CsvWriter cw2;

CrawlStat myCrawlStat;

public MyCrawler()throws Exception{

myCrawlStat = new CrawlStat();

csv = new File(CVS_PATH);

csv2 = new File(LINK_PATH);

if(csv.isFile()){

csv.delete();

}

if(csv2.isFile()){

csv2.delete();

}

cw = new CsvWriter(CVS_PATH,',', Charset.forName("GBK"));

String[] headerCw = {"图片","价格","地址","说明"};

cw.writeRecord(headerCw);

cw.close();

}

public void dumpMyData(){

final int id = getMyId();

logger.info("Crawler {} > Processed Pages: {}",id,myCrawlStat.getTotalProcessedPages());

logger.info("Crawler {} > Total Links Found: {}",id,myCrawlStat.getTotalLinks());

logger.info("Crawler {} > Total Text Size: {}",id,myCrawlStat.getTotalTextSize());

}

@Override

public Object getMyLocalData(){

return myCrawlStat;

}

@Override

public void onBeforeExit(){

dumpMyData();

}

public boolean shouldVisit(Page referringPage, WebURL url){

final String href = url.getURL().toLowerCase();

if(FILTERS.matcher(href).matches() ||!href.startsWith(URL_PREFIX)){

return false;

}

return true;

}

@Override

public void visit(Page page){

final String url = page.getWebURL().getURL();

logger.info("====爬取路径===="+url);

myCrawlStat.incProcessedPages();

if(page.getParseData() instanceof HtmlParseData){

try{

final HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();

final List<WebURL> links = htmlParseData.getOutgoingUrls();

// linkToCsv(links);

myCrawlStat.incTotalLinks(links.size());

myCrawlStat.incTotalTextSize(htmlParseData.getText().getBytes("GBK").length);

final String html = htmlParseData.getHtml();

final Document doc = Jsoup.parse(html);

final Elements contents = doc.select("li[class=clearfix]");

BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csv, true), "GBK"), 1024);

cw = new CsvWriter(out, ',');

for(final Element c : contents) {

final String img = c.select(".img img").first().attr("src");//图片

//地址

final Element txt = c.select("div[class=txt]").first();

final String arr1 = txt.select("h3 a").first().text();

final String arr2 = txt.select("h4 a").first().text();

final String arr3 = txt.select("div[class=detail]").first().text();

final String arr = arr1.concat(arr1 + ",").concat(arr2 + ",").concat(arr3);

final String rank = txt.select("p").first().text();//说明

final String pirce = c.select("p[class=price]").first().text();//价格

System.out.println("返回数据：" + img+"|"+arr+"|"+rank+"|"+pirce);

String[] content = {img,pirce,arr,rank};

/* cw.write(img);

cw.write(pirce);

cw.write(new String(arr.getBytes("utf-8"),"GBK"));

cw.write(rank);*/

cw.writeRecord(content);

}

cw.flush();

cw.close();

}catch (Exception ex){

ex.printStackTrace();}

}

}

private void linkToCsv(List<WebURL> links)throws Exception{

cw2 = new CsvWriter(LINK_PATH,',', Charset.forName("GBK"));

String[] header = {"请求路径"};

cw2.writeRecord(header);

for(final WebURL webURL:links){

String[]contentUrl = {webURL.getURL()};

cw2.writeRecord(contentUrl);

}

cw2.close();

}

}

测试的main函数：
public static void main(String[] args){

final String crawlStorageFolder = "data/crawl/root";

final int numberOfCrawlers = 2;

final CrawlConfig config = new CrawlConfig();

config.setCrawlStorageFolder(crawlStorageFolder);

config.setPolitenessDelay(1000);

config.setIncludeBinaryContentInCrawling(false);

config.setMaxPagesToFetch(50);

final PageFetcher pageFetcher = new PageFetcher(config);

final RobotstxtConfig robotstxtConfig = new RobotstxtConfig();

final RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig,pageFetcher);

CrawlController controller;

try{

controller = new CrawlController(config,pageFetcher,robotstxtServer);

controller.addSeed("http://johnhany.net/");

controller.start(MyCrawler.class,numberOfCrawlers);

final List<Object> crawlersLocalData = controller.getCrawlersLocalData();

long totalLinks = 0;

long totalTextSize = 0;

int totalProcessedPages = 0;

for(final Object localData :crawlersLocalData){

final CrawlStat stat = (CrawlStat) localData;

totalLinks +=stat.getTotalLinks();

totalTextSize +=stat.getTotalTextSize();

totalProcessedPages +=stat.getTotalProcessedPages();

}

System.out.println("Aggregated Statistics:");

System.out.print("\tProcessed Pages:{}"+totalProcessedPages);

System.out.println("\tTotal Links found: {}" + totalLinks);

System.out.println("\tTotal Text Size: {}" + totalTextSize);

}catch (Exception ex){

ex.printStackTrace();

}
}