Webmagic爬虫--②爬自己的CSDN博客列表

<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.3</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.3</version>
</dependency>

package com.xt;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
* Created by XT on 2017/11/24.
*/
public class MyCSND implements PageProcessor {
//http://blog.csdn.net/weixin_35852328/article/list/1
public static final String URL_LIST = "http://blog\\.csdn\\.net/weixin_35852328/article/list/\\d{1}";

//http://blog.csdn.net/weixin_35852328/article/details/78144353
public static final String URL_POST = "http://blog\\.csdn\\.net/weixin_35852328/article/details/\\d{8}";

private static int count = 0;

private Site site = Site
.me()
.setDomain("blog.csdn.net")
.setRetryTimes(3).setSleepTime(100);

public void process(Page page) {
//列表页
if (page.getUrl().regex(URL_LIST).match()) {
page.addTargetRequests(page.getHtml().xpath("//span[@class=link_title]/a/@href").all());
page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());
//文章页
} else {
count++;
page.putField("title", page.getHtml().xpath("//span[@class='link_title']/a/text()"));
page.putField("view", page.getHtml().xpath("//span[@class='link_view']/text()"));
page.putField("date",
page.getHtml().xpath("//span[@class='link_postdate']/text()"));
}
}

public Site getSite() {
return site;
}

public static void main(String[] args) {
long startTime, endTime;
System.out.println("开始爬取...");
startTime = System.currentTimeMillis();
Spider.create(new MyCSND()).addUrl("http://blog.csdn.net/weixin_35852328/article/list/2").thread(5).run();
endTime = System.currentTimeMillis();
System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了" + count + "条记录");

}
}