利用htmlunit——WebClient实现的java爬虫程序

package com.kinth.worker.webmagic.processors;

import java.io.IOException;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class Crawler {
public static void main(String[] args) {

WebClient webClient= new WebClient(BrowserVersion.CHROME);//设置浏览器
webClient.getOptions().setUseInsecureSSL(true);
webClient.getOptions().setCssEnabled(true);//设置css是否生效
webClient.getOptions().setJavaScriptEnabled(true);//设置js是否生效
webClient.setAjaxController(new NicelyResynchronizingAjaxController());//设置ajax请求
webClient.getOptions().setTimeout(10000);
webClient.waitForBackgroundJavaScript(3000);
HtmlPage htmlPage = null;
try {
htmlPage = webClient.getPage("https://auction.rmfysszc.gov.cn/Result.aspx");
} catch (FailingHttpStatusCodeException | IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}//访问路径设置
System.out.println(htmlPage.asXml());
webClient.close();
System.out.println("Success");
System.exit(0);
}
}