Java之网络爬虫WebCollector+selenium+phantomjs(三)

经过前面两篇的学习Java之网络爬虫WebCollector+selenium+phantomjs(一)与Java之网络爬虫WebCollector+selenium+phantomjs(二)的学习后，我们来做一个小例子。我们所要做的东西为:爬取到京东列表页面，在页面上抽取出商品信息(名称、价格、评价)，然后打印出抽取的商品信息。

贴出代码：

Goods.Java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler;

/**
*商品信息
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-21
*/
public class Goods {
private String platform;
private String url;
private String name;
private Float price;
private Integer commit;

public Goods(){
}

public String getPlatform() {
return platform;
}

public void setPlatform(String platform) {
this.platform = platform;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Float getPrice() {
return price;
}
public void setPrice(Float price) {
this.price = price;
}
public Integer getCommit() {
return commit;
}
public void setCommit(Integer commit) {
this.commit = commit;
}

@Override
public String toString() {
return "{platform="+platform+",url=" + url + ",name=" + name + ",price="
+ price + ",commit=" + commit + "}";
}

}

上面类为封装的商品信息。

EECrawler.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler;

import java.util.concurrent.atomic.AtomicInteger;

import cn.edu.hfut.dmic.webcollector.crawler.DeepCrawler;
import cn.edu.hfut.dmic.webcollector.model.Links;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.HttpResponse;
import cn.edu.hfut.dmic.webcollector.util.RegexRule;

/**
*电商平台爬虫
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-20
*/
public abstract class ECCrawler extends DeepCrawler {

private String seedFormat;//种子格式化
protected RegexRule regexRule;

public RegexRule getRegexRule() {
return regexRule;
}
public void setRegexRule(RegexRule regexRule) {
this.regexRule = regexRule;
}
public void addRegex(String urlRegex) {
this.regexRule.addRule(urlRegex);
}
public ECCrawler(String crawlPath,String seedFormat ){
super(crawlPath);
this.seedFormat=seedFormat;
this.regexRule=new RegexRule();
}

/*用一个自增id来生成唯一文件名*/
AtomicInteger id=new AtomicInteger(0);

@Override
public Links visitAndGetNextLinks(Page page) {
Links nextLinks = new Links();
String conteType = page.getResponse().getContentType();
if (conteType != null && conteType.contains("text/html")) {
org.jsoup.nodes.Document doc = page.getDoc();
if (doc != null)
nextLinks.addAllFromDocument(page.getDoc(), regexRule);
}
try {
visit(page, nextLinks);
} catch (Exception ex) {
LOG.info("Exception", ex);
}
return nextLinks;
}
@Override
public void start(int depth) throws Exception {
addSeed();
super.start(depth);
}
/**
* add seed
*
* @throws Exception
*/
private void addSeed() throws Exception{
int totalPage=getTotalPage(getPage(getSeed(seedFormat, 1)));
for(int page=1;page<=totalPage;page++){
this.addSeed(getSeed(seedFormat, page));
}
}

/**
* 根据url获取Page实例
*
* @param url
* @return
* @throws Exception
*/
private Page getPage(String url) throws Exception {
HttpRequest httpRequest = new HttpRequest(url);
HttpResponse response = httpRequest.getResponse();
Page page = new Page();
page.setUrl(url);
page.setHtml(response.getHtmlByCharsetDetect());
page.setResponse(response);
return page;
}

/**
*获取查询商品总页数
*
* @return
*/
public abstract int getTotalPage(Page page);

/**
* 获取seed url
*
* @param seedFormat
* @param page
* @return
*/
public String getSeed(String seedFormat,Object ... page){
return String.format(seedFormat, page);
}

public abstract void visit(Page page, Links links);
}

上面抽象类继承DeepCrawler，为爬取电商列表也基类，爬取列表页html（包括js动态生成的html）,并且可以抽取到列表页数，允许捕获所有页商品信息。

GoodsList.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler;

import java.util.ArrayList;

import cn.edu.hfut.dmic.webcollector.model.Page;

/**
*
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-23
*/
public abstract class GoodsList extends ArrayList<Goods> {

/**
*
*/
private static final long serialVersionUID = -6935403464055289581L;

public abstract void addGoods(Page page);
}

上面抽象类为存储商品信息的容器，继承自ArrayList,并且添加addGoods方法，用来添加商品信息到容器中。

JDCrawler.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler.jd;

import cn.edu.hfut.dmic.webcollector.model.Links;
import cn.edu.hfut.dmic.webcollector.model.Page;

import com.zhao.crawler.ECCrawler;
import com.zhao.crawler.Goods;

/**
*JD 爬虫
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-20
*/
public class JDCrawler extends ECCrawler {

private JDGoodsList goodsList;

/**
*
*
* @param crawlPath
* @param seekFormat
*/
public JDCrawler(String crawlPath, String seekFormat) {
super(crawlPath, seekFormat);
goodsList=new JDGoodsList();
}

@Override
public int getTotalPage(Page page) {
//		Element ele=page.getDoc().select("div#J_bottomPage").select("span.p-skip >em").first().select("b").first();
//		return ele==null?0:Integer.parseInt(ele.text());
return 1;
}

@Override
public void visit(Page page, Links links) {
System.out.println("url:"+page.getUrl()+"\tlinks size:"+links.size());
goodsList.addGoods(page);
}

public static void main(String[] args) throws Exception {
JDCrawler crawler=new JDCrawler("D:/test/crawler/jd/", "http://list.jd.com/list.html?cat=1319,1523,7052&page=%s&go=0&JL=6_0_0");
crawler.setThreads(100);//抓取启动线程数
crawler.start(1);//层数

crawler.print();
}

protected void print(){
for(Goods g:goodsList){
System.out.println(g);
}
}
}

继承ECCrawler，实现京东平台专属爬取类。获取页码数利用浏览器审查元素，定位到页面信息即可，为了方便测试，这里只返回1。启动时我们直接爬取种子页面，所以设置为1即可，具体的抽取商品信息交给了下面JDGoodsList来处理。抓取结束后，执行一边打印函数，打印出商品信息。

JDGoodsList.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler.jd;

import java.util.List;

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;

import cn.edu.hfut.dmic.webcollector.model.Page;

import com.zhao.crawler.Goods;
import com.zhao.crawler.GoodsList;
import com.zhao.crawler.util.PageUtils;
import com.zhao.crawler.util.Platform;
import com.zhao.crawler.util.Tools;

/**
*
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-23
*/
public class JDGoodsList extends GoodsList {

/**
*
*/
private static final long serialVersionUID = -7487110223660262262L;

@Override
public void addGoods(Page page) {
WebDriver driver = null;
try {
driver = PageUtils.getWebDriver(page);
List<WebElement> eles = driver.findElements(By.cssSelector("li.gl-item"));
if (!eles.isEmpty()) {
for (WebElement ele : eles) {
Goods g = new Goods();
g.setPlatform(Platform.JD);// 电商平台
// 价格
String priceStr = ele.findElement(By.className("p-price"))
.findElement(By.className("J_price"))
.findElement(By.tagName("i"))
.getText();
if (Tools.notEmpty(priceStr)) {
g.setPrice(Float.parseFloat(priceStr));
} else {
g.setPrice(-1f);
}
// 商品名
g.setName(ele.findElement(By.className("p-name"))
.findElement(By.tagName("em")).getText());
// 商品链接
g.setUrl(ele.findElement(By.className("p-name"))
.findElement(By.tagName("a"))
.getAttribute("href"));
// 评价
String commitStr = ele
.findElement(By.className("p-commit"))
.findElement(By.tagName("a"))
.getText();
if (Tools.notEmpty(commitStr)) {
g.setCommit(Integer.parseInt(commitStr));
} else {
g.setCommit(-1);
}

add(g);
}
} else {
System.out.println("else is empty");
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (driver != null) {
driver.quit();
}
}
}
}

PageUtils.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
package com.zhao.crawler.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.openqa.selenium.WebDriver;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriver;

import com.gargoylesoftware.htmlunit.BrowserVersion;

import cn.edu.hfut.dmic.webcollector.model.Page;

/**
*
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-22
*/
public class PageUtils {

/**
* 获取webcollector 自带 htmlUnitDriver实例(模拟默认浏览器)
*
* @param page
* @return
*/
public static HtmlUnitDriver getDriver(Page page) {
HtmlUnitDriver driver = new HtmlUnitDriver();
driver.setJavascriptEnabled(true);
driver.get(page.getUrl());
return driver;
}

/**
* 获取webcollector 自带htmlUnitDriver实例
*
* @param page
* @param browserVersion 模拟浏览器
* @return
*/
public static HtmlUnitDriver getDriver(Page page,
BrowserVersion browserVersion) {
HtmlUnitDriver driver = new HtmlUnitDriver(browserVersion);
driver.setJavascriptEnabled(true);
driver.get(page.getUrl());
return driver;
}

/**
* 获取PhantomJsDriver(可以爬取js动态生成的html)
*
* @param page
* @return
*/
public static WebDriver getWebDriver(Page page) {
// WebDriver driver = new HtmlUnitDriver(true);

// System.setProperty("webdriver.chrome.driver", "D:\\Installs\\Develop\\crawling\\chromedriver.exe");
// WebDriver driver = new ChromeDriver();

System.setProperty("phantomjs.binary.path", "D:/Program Files/phantomjs-2.0.0-windows/bin/phantomjs.exe");
WebDriver driver = new PhantomJSDriver();
driver.get(page.getUrl());

// JavascriptExecutor js = (JavascriptExecutor) driver;
// js.executeScript("function(){}");
return driver;
}

/**
* 直接调用原生phantomJS(即不通过selenium)
*
* @param page
* @return
*/
public static String getPhantomJSDriver(Page page) {
Runtime rt = Runtime.getRuntime();
Process process = null;
try {
process = rt.exec("D:/Program Files/phantomjs-2.0.0-windows/bin/phantomjs.exe" +
"D:/MyEclipseWorkSpace/WebCollectorDemo/src/main/resources/parser.js " +
page.getUrl().trim());
InputStream in = process.getInputStream();
InputStreamReader reader = new InputStreamReader(
in, "UTF-8");
BufferedReader br = new BufferedReader(reader);
StringBuffer sbf = new StringBuffer();
String tmp = "";
while((tmp = br.readLine())!=null){
sbf.append(tmp);
}
return sbf.toString();
} catch (IOException e) {
e.printStackTrace();
}

return null;
}
}


获取WebDriver工具类，上篇有介绍。

Platform.java

/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/
package com.zhao.crawler.util;

/**
*电商平台标识
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-23
*/
public interface Platform {
/**
* 京东
*/
public static final String JD="JD";
}

Tools.java
/*
* Copyright (C) 2015 zhao
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
package com.zhao.crawler.util;

import org.apache.commons.lang3.StringUtils;

/**
*
*
* @author <a href="ls.zhaoxiangyu@gmail.com">zhao</>
* @date 2015-10-23
*/
public class Tools {

/**
* 判断字符窜是否等于null、""," ","null"
*
* @param str
* @return
*/
public static boolean isEmpty(String str){
return StringUtils.isBlank(str)||"null".equals(str);
}

/**
* 判断字符窜是否不等于null、""," ","null"
*
* @param str
* @return
*/
public static boolean notEmpty(String str){
return !StringUtils.isBlank(str)&&!"null".equals(str);
}

}


运行程序，控制台输出结果为:



ok,成功抽取商品信息。

自此，此次学习结束。源码下载地址(免费下载):WebCollectorDemo