crawler4j源码学习(1)：搜狐新闻网新闻标题采集爬虫

/**
* @date 2016年8月20日 上午11:52:13
* @version
* @since JDK 1.8
*/
public class MyCrawler extends WebCrawler {

//链接地址过滤//
private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$");

@Override
public boolean shouldVisit(Page referringPage, WebURL url) {
String href = url.getURL().toLowerCase();
return !FILTERS.matcher(href).matches() && href.startsWith("http://news.sohu.com/");
}

/**
* This function is called when a page is fetched and ready to be processed
* by your program.
*/
@Override
public void visit(Page page) {
String url = page.getWebURL().getURL();
logger.info("URL: " + url);

if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
String text = htmlParseData.getText();
String html = htmlParseData.getHtml();
Set<WebURL> links = htmlParseData.getOutgoingUrls();

logger.debug("Text length: " + text.length());
logger.debug("Html length: " + html.length());
logger.debug("Number of outgoing links: " + links.size());
logger.info("Title: " + htmlParseData.getTitle());

}
}

}

/**
* @date 2016年8月20日 上午11:55:56
* @version
* @since JDK 1.8
*/
public class MyController {

/**
* @param args
* @since JDK 1.8
*/
public static void main(String[] args) {
// TODO Auto-generated method stub

//本地嵌入式数据库，采用berkeley DB
String crawlStorageFolder = "data/crawl/root";
int numberOfCrawlers = 3;

CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorageFolder);

/*
* Instantiate the controller for this crawl.
*/
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller;
try {
controller = new CrawlController(config, pageFetcher, robotstxtServer);
controller.addSeed("http://news.sohu.com/");
controller.start(MyCrawler.class, numberOfCrawlers);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

}