htmlUnit的使用

package com.swpu;

import java.io.IOException;
import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSubmitInput;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;

public class WorldBankCrawl {

public static void main(String[] args)
throws FailingHttpStatusCodeException, MalformedURLException,
IOException {
final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_38);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setJavaScriptEnabled(false); // 必须加上
HtmlPage page = webClient
.getPage("https://www.baidu.com/?tn=96010190_dg");

final HtmlForm form = page.getFormByName("f");
final HtmlSubmitInput submitInput = form.getInputByValue("百度一下");
final HtmlTextInput input = form.getInputByName("wd");
input.setValueAttribute("西游记");

final HtmlPage nextPage = (HtmlPage) submitInput.click();
// String nextString = nextPage.asText();
// System.out.println(nextString);
// System.out.println(nextPage.asXml());
final java.util.List<?> images = nextPage
.getByXPath("//a[@title='万圣公主']/../..//img");
for (Object image : images) {
System.out.println(image);
}
System.out.println("src:\n");
System.out.println(nextPage
.getByXPath("//a[@title='万圣公主']/../..//@src"));
System.out.println(nextPage
.getByXPath("//a[@title='万圣公主']/../..//@title"));
System.out.println(nextPage.getByXPath(
"//a[@title='万圣公主']/../..//@title").size());

final java.util.List<?> table = nextPage
.getByXPath("//table[@class='c-table opr-toplist-table']");
System.out.println("test table:");
System.out.println(table);

final java.util.List<?> link = nextPage
.getByXPath("//a[@class='n']//@href");

System.out.println("link:" + link);
HtmlAnchor next2Anchor = (HtmlAnchor) nextPage.getByXPath(
"//a[@class='n']").get(0);
HtmlPage next2Page = next2Anchor.click();
System.out.println("NEXT PAGE:\n");
System.out.println(next2Page.asText());
System.out.println("测试XPath函数：");
System.out.println(next2Page
.getByXPath("//span[@title='《西游记》人物']/text()"));
System.out.println(next2Page.getByXPath("//span[@title='《西游记》人物']"));
System.out.println(next2Page.getByXPath("//div[2]"));
System.out.println("函数设置：");
System.out.println(next2Page.getByXPath("//a[@class='n']"));

// final HtmlTableHeader header = ((HtmlTable) table).getHeader();
// final List<HtmlTableRow> headerRows = header.getRows();
// final HtmlTableHeader header = ((HtmlTable) table).getHeader();
/*
* final java.util.List<HtmlTableRow> headerRows = header.getRows();
* for(HtmlTableRow headerRow : headerRows) {
* System.out.println(headerRow.asText()); }
*/

/*
* final java.util.List<?> attributeList =
* nextPage.getByXPath("//a[@title='万圣公主']/../..//@src;"); for(Object
* attr: attributeList) { System.out.println(attr); }
*/
// System.out.println(
// nextPage.getByXPath("//a[@title='万圣公主']/../..//img").get(0));

// System.out.println(nextPage.getByXPath("//a[@title='万圣公主']/../..//img"));
// //不需要特定顺序必须使用//
}
}

package com.swpu;

import java.io.IOException;
import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class ParseSinaBlog {

public static void main(String[] args)
throws FailingHttpStatusCodeException, MalformedURLException,
IOException {
// TODO Auto-generated method stub

try {

final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_38);

webClient.getOptions().setJavaScriptEnabled(true); // 必须设置为true
webClient.getOptions().setCssEnabled(false);
webClient
.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setThrowExceptionOnScriptError(false);

final HtmlPage page = (HtmlPage) webClient
.getPage("http://login.sina.com.cn/sso/login.php?"
+ "client=ssologin.js(v1.3.16)");

HtmlInput ln = page.getHtmlElementById("username");
HtmlInput pwd = page.getHtmlElementById("password");
HtmlInput btn = page.getFirstByXPath(".//*[@id='vForm']/"
+ "div[3]/ul/li[6]/div[2]/input");

ln.setAttribute("value", "你的用户名");
pwd.setAttribute("value", "你的密码");

HtmlPage page2 = btn.click();
// 登录完成，现在可以爬取任意你想要的页面了。

// System.out.println(page2.asText());
/*
* HtmlAnchor anchor = (HtmlAnchor)page.getByXPath
* ("//li/a[@href='http://weibo.com']");
*/
/*
* System.out.println("anchor:\n\n\n" + page.getByXPath
* ("//li/a[@href='http://weibo.com']"));
*/
/*
* HtmlPage weiboPage = anchor.click(); java.util.List<?> wbList =
* weiboPage.getByXPath ("//div[@class]"); // \"WB_text W_f14\
*
* for(Object object : wbList) { System.out.println(object); }
*/

/*
* HtmlPage nextPage = webClient.getPage("http://weibo.com/friends?"
* + "leftnav=1&wvr=6&isfriends=1&step=2");
* System.out.println(nextPage.asXml());
*/

/*
* HtmlPage page3 = webClient.getPage("http://weibo.com/" +
* "friends?leftnav=1&wvr=5&isfriends=1&step=2");
*/
// System.out.println(" : " + page3.asXml());
HtmlPage firstPage = webClient.getPage("http://weibo.com/"
+ "u/2795493364/home?leftnav=1");
System.out.println(firstPage.asXml());

/*
* java.util.List<?> contents = firstPage.getByXPath
* ("//div[@class='WB_text W_f14']");
*
* System.out.println("print contents:"); for(Object content :
* contents) { System.out.println(content);
*
*
* }
*/

} catch (Exception ex) {
System.out.println(ex.getMessage());
}

}
}