java解析html之HTMLparser初次尝试

1. HtmlParser 简介

2. 建立Maven工程

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>
<groupId>com.fancy</groupId>
<artifactId>htmlParser</artifactId>
<version>0.0.1-SNAPSHOT</version>

<dependencies>
<dependency>
<groupId>org.htmlparser</groupId>
<artifactId>htmlparser</artifactId>
<version>2.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

2.1 创建一个解析器

org.htmlparser

Interface Node

 void

accept(NodeVisitor visitor)

 Object

clone()

 void

collectInto(NodeList list,NodeFilter filter)

 void

doSemanticAction()

 NodeList

getChildren()

 int

getEndPosition()

 Node

getFirstChild()

 Node

getLastChild()

 Node

getNextSibling()

 Page

getPage()

 Node

getParent()

 Node

getPreviousSibling()

 int

getStartPosition()

 String

getText()

 void

setChildren(NodeList children)

 void

setEndPosition(int position)

 void

setPage(Page page)

 void

setParent(Node node)

 void

setStartPosition(int position)

 void

setText(String text)

 String

toHtml()

 String

toHtml(boolean verbatim)

 String

toPlainTextString()

 String

toString()

抓取http://www.v2ex.com网站中的一篇帖子

/**
*
* 获取html中的主题和所有回复节点
*
* @param url
* @param ENCODE
* @return
*/
protected  NodeList getNodelist(String url, String ENCODE) {

try {
NodeList nodeList = null;
Parser parser = new Parser(url);
parser.setEncoding(ENCODE);
//定义一个Filter，过滤主题div
NodeFilter filter = new NodeFilter() {
@Override
public boolean accept(Node node) {
if(node.getText().contains("style=\"border-bottom: 0px;\"")) {
return true;
} else {
return false;
}
}
};
//定义一个Filter，过滤所有回复div
NodeFilter replyfilter = new NodeFilter() {
@Override
public boolean accept(Node node) {
String containsString = "id=\"r_";
if(node.getText().contains(containsString)) {
return true;
} else {
return false;
}
}
};
//组合filter
OrFilter allFilter = new OrFilter(filter, replyfilter);

nodeList = parser.extractAllNodesThatMatch(allFilter);
return nodeList;

} catch (ParserException e) {
e.printStackTrace();
return null;
}

}

public Forum parse2Thread(String url,String ENCODE) {
List<Reply> replylist = new ArrayList<Reply>();	//回复列表
Topic topic = new Topic();	//主题
NodeFilter divFilter = new NodeClassFilter(Div.class);//div过滤器
NodeFilter headingFilter = new NodeClassFilter(HeadingTag.class);//heading过滤器
NodeFilter tagFilter = new NodeClassFilter(TagNode.class);//heading过滤器

NodeList nodeList = this.getNodelist(url, ENCODE);

//解析node到帖子实体
for (int i = 0; i < nodeList.size(); i++) {
Node node = nodeList.elementAt(i);
if(node.getText().contains("style=\"border-bottom: 0px;\"")) {
//如果node是主题
NodeList list = node.getChildren();//node的子节点
//header div
Node headerNode = list.extractAllNodesThatMatch(new NodeClassFilter(Div.class)).elementAt(0);
//帖子主题
Node h1Node = headerNode.getChildren().extractAllNodesThatMatch(headingFilter).elementAt(0);
topic.setTopicName(h1Node.toPlainTextString());
//发帖人信息
NodeList headerChrildrens = headerNode.getChildren();
topic.setAnn_name(headerChrildrens.elementAt(15).toPlainTextString());
topic.setTopicDescribe(headerChrildrens.elementAt(16).toPlainTextString());
//发帖人头像链接
Node frNode = headerChrildrens.extractAllNodesThatMatch(divFilter).elementAt(0);
ImageTag imgNode = (ImageTag) frNode.getFirstChild().getFirstChild();
topic.setAnn_img(imgNode.getImageURL());

//cell div
Node cellNode = list.extractAllNodesThatMatch(divFilter).elementAt(1);
Node topic_content = cellNode.getChildren().extractAllNodesThatMatch(divFilter).elementAt(0);
Node markdown_body = topic_content.getChildren().extractAllNodesThatMatch(divFilter).elementAt(0);
topic.setTopicBody(markdown_body.toPlainTextString());//暂时不包含连接和图片纯文本

} else if(node.getText().contains("id=\"r_")){
//节点是回复
Reply reply = new Reply();

Node tableNode = node.getChildren().extractAllNodesThatMatch(tagFilter).elementAt(0);
Node trNode = tableNode.getChildren().extractAllNodesThatMatch(tagFilter).elementAt(0);
//回复的tagNodeList
NodeList tagList = trNode.getChildren().extractAllNodesThatMatch(tagFilter);
ImageTag reply_img = (ImageTag) tagList.elementAt(0).getChildren().extractAllNodesThatMatch(tagFilter).elementAt(0);
reply.setReply_img(reply_img.getImageURL());
//nodeList bodyNode = tagList;
replylist.add(reply);
}
}
System.out.println("-----------实体----------------");
Forum forum = new Forum(topic, replylist);
System.out.println(forum.toString());

return null;
}

@Test
public  void test() throws Exception {
Html2Domain parse = new Html2DomainImpl();
parse.parse2Thread("http://www.v2ex.com/t/262409#reply6","UTF-8");

}