使用webmagic爬取搜狗上公众账号发布的文章信息

/**

* 爬取文章类

*/

public class ArticleCrawler {

Static ApplicationContext context = SpringFactory.getApplicationContext();
Static ArticleDao articleDao=(ArticleDao)context.getBean("articleDao");
Static AccountCrawlerDao accountCrawlerDao = (AccountCrawlerDao)context.getBean("accountCrawlerDao");

/**
* 爬取文章信息
* @param pageUrl
* @return
* @throws ParseException
*/
public ArrayList<ArticleBo> getPublicArticles(String pageUrl){
ArrayList<ArticleBo> boList = new ArrayList<>();
Spider spider = Spider.create(new ArticleCrawlerProcessor())
.thread(3).addPipeline(new ResultItemsCollectorPipeline());

ResultItems resultItems = spider.get(pageUrl);
spider.close();

boList=resultItems.get("articleList");
return boList;
}

/**
* 爬取＋入库流程
*/
public void run(String openId){
// 得到公众号发布文章信息
ArrayList<ArticleBo> articleList = new ArrayList<ArticleBo>();

String Url = "http://weixin.sogou.com/gzhjs?openid="+openId;

// 爬取文章
articleList = getPublicArticles(Url);

if(null!=articleList&&articleList.size()>0){
// 文章入库
for(ArticleBo bo:articleList){
try {
articleDao.addArticle(bo);
} catch (Exception e){
// 入库失败
}
}
}
}

public static void main(String args[]){
ArticleCrawler crawler = new ArticleCrawler();

// 得到需要爬取发布文章的公众账号
List<AccountCrawlerBo> boList = accountCrawlerDao.getCrawlerList();
if (null!=boList&&boList.size()>0) {
for (AccountCrawlerBo bo:boList) {
crawler.run(bo.getPublicOpenId());
}
｝
}

}

/**

* 爬虫进程类

*/

public class ArticleCrawlerProcessor implements PageProcessor {

private Site site = Site.me().setRetryTimes(3).setSleepTime(60000);

@Override
public void process(Page page) {

// 文章列表对应的审查元素
List<String> resultList = page.getHtml().xpath("//item").all();

List<ArticleBo> articleList = new ArrayList<>();
if((resultList!=null)&&(resultList.size()>0))
{
// 解析文章信息：title,biz,mid,url,abstract，createTime等
for(int i=0; i<resultList.size(); i++){

Html html = new Html(resultList.get(i));

String title = StringTool.getMiddleStr("<![CDATA[","]]>",html.xpath("title/text()").get());
String url = html.xpath("url/text()").get();
String biz = StringTool.getMiddleStr("__biz=","&mid=",url);
String mid = StringTool.getMiddleStr("&mid=","&idx=",url);
int idx = Integer.parseInt(StringTool.getMiddleStr("&idx=", "&sn=", url));
String sn = StringTool.getMiddleStr("&sn=","&3rd=",url);
String publicOpenId = StringTool.getMiddleStr(" ","<",html.xpath("openid/text()").get());
String articleAbstract = StringTool.getMiddleStr(" ","<", html.xpath("content168/text()").get());
String articleAbstractPicture = StringTool.getMiddleStr(" ","<", html.xpath("imglink/text()").get());
String articleCreatetime = StringTool.getMiddleStr(" ", "<", html.xpath("lastmodified/text()").get());
articleCreatetime = StringTool.getTimeFromUnix(articleCreatetime);

ArticleBo bo = new ArticleBo();

bo.setArticleTitle(title);
bo.setArticleUrl(url);
bo.setArticleBiz(biz);
bo.setArticleMid(mid);
bo.setArticleIdx(idx);
bo.setArticleSn(sn);
bo.setPublicOpenId(publicOpenId);
bo.setArticleAbstract(articleAbstract);
bo.setArticleAbstractPicture(articleAbstractPicture);
bo.setArticleCreatetime(articleCreatetime);

articleList.add(bo);
}
}

page.putField("articleList", articleList);
}

@Override
public Site getSite() {
return site;
}

}