Nutch学习笔记13---以某网站为例写解析插件

编写自己的HtmlParseFilter---sohu

1 进入到文件夹 $NUTCH_HOME/src/plugin

mkdir htmlparsefilter-sohu

2 按照下面的结构建立目录和文件

htmlparsefilter-sohu/

plugin.xml

build.xml

ivy.xml

src/

java/

org/

apache/

nutch/

parse/

HtmlParseFilterSohu.java

3 修改plugin.xml

<?xml version="1.0" encoding="UTF-8"?>

<plugin id="htmlparsefilter-sohu" name="Add Sohu Field to Doc"

version="1.0.0" provider-name="nutch.org">

<runtime>

<library name="htmlparsefilter-sohu.jar">

<export name="*"/>

</library>

</runtime>

<requires>

</requires>

<extension id="org.apache.nutch.parse.parse_sohu"

name="Add sohu Field to doc"

point="org.apache.nutch.parse.HtmlParseFilter">

<implementation id="HtmlParseFilterSohu"

class="org.apache.nutch.parse.HtmlParseFilterSohu"/>

</extension>

</plugin>

4 关于ivy.xml

从plugin/index-basic下面复制对应的ivy.xml，不需要任何改变即可。

5 关于build.xml

修改成以下内容

<?xml version="1.0" encoding="UTF-8"?>

<project name="htmlparsefilter-sohu" default="jar">

<import file="../build-plugin.xml"/>

</project>

6 修改 HtmlParseFilterSohu.java-以实际需求为准-以实际代码为准

7 修改src/plugin/build.xml

找到

<!-- ====================================================== -->

<!-- Build & deploy all the plugin jars. -->

<!-- ====================================================== -->

在下面添加一行

<ant dir="htmlparsefilter-sohu" target="deploy"/>

8 修改nutch-site.xml

我的运行在local模式下，则修改配置文件local/conf/nutch-site.xml如下

从nutch-default.xml中复制plugin.includes的配置块

<property>

<name>plugin.includes</name>

<value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)</value>

<description> </description>

</property>

到nutch-site.xml中

然后修改复制后的内容

<property>

<name>plugin.includes</name>

<value>protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-solr|scoring-opic|urlnormalizer-(pass|regex|basic)|htmlparsefilter-sohu</value>

<description> </description>

</property>

就可以了。

9 最后一步-修改$NUTCH_HOME/conf/schema.xml

在<fields>...</fields>段内添加

<field name="pageLength"type="long"stored="true"indexed="true"/>

10 重新ant , 大功告成。

java文件如下：

package org.apache.nutch.parse;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.protocol.Content;
import org.w3c.dom.Comment;
import org.w3c.dom.Document;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;

public class HtmlParseFilterSohu implements HtmlParseFilter {

private static final Log LOG = LogFactory.getLog(HtmlParseFilterSohu.class);
private Configuration conf;
private ParseResult _parseResult;
private Content _content;

private static final String A = "a";
private static final String ACTOR = "actor";
private static final String ALBUM = "album";
private static final String AREABOX = "areabox";
private static final String CATEGORY = "category";
private static final String CLASS = "class";
private static final String CONTENT = "content";
private static final String CONTENT_LOCATION = "contentLocation";
private static final String CRUMBS = "crumbs";
private static final String CRUMBSBAR = "crumbsBar";
private static final String DATA_SUBSCRIBE_CATEGORYNAME = "data-subscribe-categoryname";
private static final String DATE_PUBLISHED = "datePublished";
private static final String DESCRIPTION = "description";
private static final String DIRECTOR = "director";
private static final String DIV = "div";
private static final String DURATION = "duration";
private static final String FULLDESCRIPTION = "full_desc";
private static final String GENRE = "genre";
private static final String H = "h";
private static final String H2 = "h2";
private static final String ID = "id";
private static final String INTRO = "intro";
private static final String INFO_INFO_CON = "info info-con";
private static final String IRALBUMNAME = "irAlbumName";
private static final String IRCATEGORY = "irCategory";
private static final String IRTITLE = "irTitle";
private static final String ITEM = "item";
private static final String ITEMPROP = "itemprop";
private static final String KEYWORDS = "keywords";
private static final String LABEL = "label";
private static final String LI = "li";
private static final String LINK = "link";
private static final String MAINACTOR = "mainactor";
private static final String NAME = "name";
private static final String OG_IMAGE = "og:image";
private static final String P = "p";
private static final String PEOS_INFO = "peos-info";
private static final String PROPERTY = "property";
private static final String PUB = "pub";
private static final String S_H = "s h";
private static final String SCRIPT = "script";
private static final String SERIES = "series";
private static final String SPAN = "span";
private static final String STYLE = "style";
private static final String THUMB_NAIL_URL = "thumbnailUrl";
private static final String TITLE = "title";
private static final String TYPE = "type";
private static final String MVINFO = "mvInfo"; // just a mv
private static final String META = "meta";
private static final String UL = "ul";
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~url about
private static final String URL_DOMAIN = ".sohu.com/";
private static final String URL_SUFFIX = ".shtml";
private static final String URL_SLASH = "/";

// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~parse_data
public static final String NUTCH_VIDEO_FULL_DESCRIPTION = "desc";
public static final String NUTCH_VIDEO_SERIES = "series";
public static final String NUTCH_VIDEO_YEAR = "year";
public static final String NUTCH_VIDEO_AREA = "area";
public static final String NUTCH_VIDEO_TYPE = "type";
public static final String NUTCH_VIDEO_DIRECTOR = "director";
public static final String NUTCH_VIDEO_ACTOR = "actor";
public static final String NUTCH_VIDEO_TITLE = "title";
public static final String NUTCH_VIDEO_CHANNEL = "channel";
public static final String NUTCH_VIDEO_KEYWORD = "keyword";
public static final String NUTCH_VIDEO_URL = "url";
public static final String NUTCH_VIDEO_PICTURE = "picture";
public static final String NUTCH_VIDEO_WEBSITE = "website";
public static final String NUTCH_VIDEO_TIMESPAN = "timespan";

private boolean isNumber(String segment) {
int length = segment.length();
int index = 0;
for (index = 0; index < length; index++) {
char c = segment.charAt(index);
if (c >= '0' && c <= '9') {

} else {
break;
}
}
if (index >= length) {
return true;
} else {
return false;
}
}

private boolean isValidStr(String segment) {
int length = segment.length();
int index = 0;
for (; index < length; index++) {
char c = segment.charAt(index);
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9')) {

} else {
break;
}
}
if (index >= length) {
return true;
} else {
return false;
}
}

private boolean isValidUrl(String url) {
if (null == url) {
return false;
}
int firstIndex, secondIndex, thirdIndex;

firstIndex = url.indexOf(URL_DOMAIN);
if (-1 == firstIndex) {
return false;
}
firstIndex += URL_DOMAIN.length();

secondIndex = url.indexOf(URL_SLASH, firstIndex);
if (-1 == secondIndex) {
return false;
}
int length = secondIndex - firstIndex;
if (8 != length) {
return false;
}
String str = url.substring(firstIndex, secondIndex);
if (false == this.isNumber(str)) {
return false;
}

secondIndex += URL_SLASH.length();
thirdIndex = url.indexOf(URL_SUFFIX, secondIndex);
if (-1 == thirdIndex) {
return false;
}
str = url.substring(secondIndex, thirdIndex);
if (false == this.isValidStr(str)) {
return false;
}

return true;
}

private String getKeyValue(String key) {
if (null == _parseResult || null == _content
|| null == _content.getUrl()) {
return null;
}

ParseData parseData = _parseResult.get(_content.getUrl()).getData();
if (null == parseData) {
return null;
}
Metadata metadata = parseData.getParseMeta();
if (null == metadata) {
return null;
}
return metadata.get(key);
}

/*
* private void setKeyValue(String key, String value) { if (null ==
* _parseResult || null == _content || null == _content.getUrl()) { return;
* }
*
* ParseData parseData = _parseResult.get(_content.getUrl()).getData(); if
* (null == parseData) { return; } Metadata metadata =
* parseData.getParseMeta(); if (null == metadata) { return; }
* metadata.set(key, value); }
*/

private void appendKeyValue(String key, String value) {
// used when exist more block...
if (null == _parseResult || null == _content
|| null == _content.getUrl()) {
return;
}

ParseData parseData = _parseResult.get(_content.getUrl()).getData();
if (null == parseData) {
return;
}
Metadata metadata = parseData.getParseMeta();
if (null == metadata) {
return;
}
String oldValue = metadata.get(key);
if (null == oldValue) {
metadata.set(key, value);
} else {
metadata.set(key, oldValue + " " + value);
}
LOG.info("[" + key + "]   [" + metadata.get(key) + "]");

}

private String replaceWithRegex(String str, String regEx, String replace) {
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(str);
str = m.replaceAll(replace).trim();
return str;
}

private Element getFirstChildNodeWithTagName(Element parent, String tagName) {
Element result = null;
if (null == parent) {
return null;
}
// not include itself...
NodeList nodeList = parent.getChildNodes();
int index = 0;
for (index = 0; index < nodeList.getLength(); index++) {
Node child = nodeList.item(index);
if (child instanceof Element) {
Element childElement = (Element) child;
String childTagName = childElement.getTagName();
if (null != childTagName) {
childTagName = childTagName.toLowerCase().trim();
if (childTagName.equals(tagName)) {
return childElement;
}
}
}
}
return null;
}

private Element getFirstChildNodeWithTagNameFixedAttributeLength(
Element parent, String tagName, int n) {
Element result = null;
if (null == parent) {
return null;
}
// not include itself...
NodeList nodeList = parent.getChildNodes();
int index = 0;
for (index = 0; index < nodeList.getLength(); index++) {
Node child = nodeList.item(index);
if (child instanceof Element) {
Element childElement = (Element) child;
String childTagName = childElement.getTagName();
if (null != childTagName) {
childTagName = childTagName.toLowerCase().trim();
if (childTagName.equals(tagName)) {
if (childElement.getAttributes().getLength() == n)
return childElement;
}
}
}
}
return null;
}

private Element getFirstChildNodeWithAttribute(Element parent,
String attributeName, String attributeValue) {
Element result = null;
if (null == parent) {
return null;
}
// not include itself...
NodeList nodeList = parent.getChildNodes();
int index = 0;
for (index = 0; index < nodeList.getLength(); index++) {
Node child = nodeList.item(index);
if (child instanceof Element) {
Element childElement = (Element) child;
String value = childElement.getAttribute(attributeName);
if (null != value) {
value = value.trim();
if (value.equals(attributeValue)) {
return childElement;
}
}
}
}
return null;
}

private Element getFirstDescendantWithAttribute(Node parent,
String attributeName, String attributeValue) {
// include itself...
Element result = null;
if (null == parent) {
return null;
}
if (parent instanceof Element) {
Element element = (Element) parent;
String value = element.getAttribute(attributeName);
if (null != value && value.equals(attributeValue)) {
return element;
}
}
// find from all child...
NodeList children = parent.getChildNodes();
int index = 0;
for (index = 0; index < children.getLength(); index++) {
Node child = children.item(index);
Element found = getFirstDescendantWithAttribute(child,
attributeName, attributeValue);
if (null != found) {
return found;
}
}
// not find...
return null;
}

private Element getFirstDescendantWithTag(Node parent, String tagName) {
// include itself...
Element result = null;
if (null == parent) {
return null;
}
if (parent instanceof Element) {
Element element = (Element) parent;
String value = element.getTagName();
if (null != value && value.toLowerCase().equals(tagName)) {
return element;
}
}
// find from all child...
NodeList children = parent.getChildNodes();
int index = 0;
for (index = 0; index < children.getLength(); index++) {
Node child = children.item(index);
Element found = getFirstDescendantWithTag(child, tagName);
if (null != found) {
return found;
}
}
// not find...
return null;
}

private Element getFirstDescendantWithTagPlusAttribute(Node parent,
String tagName, String attributeName, String attributeValue) {
// include itself...
Element result = null;
if (null == parent) {
return null;
}
if (parent instanceof Element) {
Element element = (Element) parent;
String tag = element.getTagName();
if (null != tag && tag.toLowerCase().equals(tagName)) {
String attrValue = element.getAttribute(attributeName);
if (null != attrValue && attrValue.equals(attributeValue)) {
return element;
}
}
}
// find from all child...
NodeList children = parent.getChildNodes();
int index = 0;
for (index = 0; index < children.getLength(); index++) {
Node child = children.item(index);
Element found = getFirstDescendantWithTagPlusAttribute(child,
tagName, attributeName, attributeValue);
if (null != found) {
return found;
}
}
// not find...
return null;
}

private Element getFirstDescendantWithTagPlusAttributeFixedAttribute(
Node parent, String tagName, String attributeName,
String attributeValue, int n) {
// include itself...
Element result = null;
if (null == parent) {
return null;
}
if (parent instanceof Element) {
Element element = (Element) parent;
String tag = element.getTagName();
if (null != tag && tag.toLowerCase().equals(tagName)) {
String attrValue = element.getAttribute(attributeName);
if (null != attrValue && attrValue.equals(attributeValue)) {

if (null == element.getAttributes()) {
if (element.getAttributes().getLength() == n)
return element;
}

}
}
}
// find from all child...
NodeList children = parent.getChildNodes();
int index = 0;
for (index = 0; index < children.getLength(); index++) {
Node child = children.item(index);
Element found = getFirstDescendantWithTagPlusAttribute(child,
tagName, attributeName, attributeValue);
if (null != found) {
return found;
}
}
// not find...
return null;
}

private void walk(Node node) {
if (null == node) {
return;
}
short nodeType = node.getNodeType();
if (nodeType == Node.DOCUMENT_FRAGMENT_NODE) {

NodeList children = node.getChildNodes();
for (int i = 0; children != null && i < children.getLength(); i++) {
walk(children.item(i));
}

} else if (nodeType == Node.ELEMENT_NODE) {

Element element = (Element) node;
String tag = element.getTagName();
if (null == tag) {
return;
}
tag = tag.toLowerCase();
if (SCRIPT.equals(tag) || STYLE.equals(tag) || LINK.equals(tag)) {
// <script>...</script>
// <style>...</style>
// <link>...</link>
return;
} else if (META.equals(tag)) {
String value = element.getAttribute(NAME);

if (null != value && value.equals(ALBUM)) {
// album
value = element.getAttribute(CONTENT);
if (null != value && value.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_SERIES, value);
}
} else if (null != value && value.equals(CATEGORY)) {
// category
value = element.getAttribute(CONTENT);
if (null != value && value.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_CHANNEL, value);
}
}

value = element.getAttribute(PROPERTY);
// image
if (null != value && value.equals(OG_IMAGE)) {
value = element.getAttribute(CONTENT);
if (null != value && value.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_PICTURE, value);
}
}

} else if (DIV.equals(tag)) {

String value = element.getAttribute(ID);
if (null != value && value.equals(CRUMBSBAR)) {
Element h2 = this.getFirstDescendantWithTag(element, H2);
if (null != h2) {
String text = h2.getTextContent();
if (null != text && text.length() > 0) {
text = this.replaceWithRegex(text, "[\\s\\t/()]+",
" ");
text = text.trim();
if (text.length() > 0)
this.appendKeyValue(NUTCH_VIDEO_TITLE, text);
}
}

return;
}

// ///////////////////////////////////////////////////////////////

value = element.getAttribute(CLASS);
if (null != value && value.equals(INFO_INFO_CON)) {

Element mainactor = this
.getFirstDescendantWithTagPlusAttribute(element,
LI, ID, MAINACTOR);
if (null != mainactor) {
Element actor = this.getFirstChildNodeWithTagName(
mainactor, A);
while (null != actor) {
String name = actor.getTextContent();
if (null != name && name.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_ACTOR, name);
}
mainactor.removeChild(actor);
actor = this.getFirstChildNodeWithTagName(
mainactor, A);
}
}

Element li = this.getFirstDescendantWithTagPlusAttribute(
element, LI, CLASS, H);
if (null != li) {
Element a = this.getFirstChildNodeWithTagName(li, A);
if (null != a) {
String year = a.getTextContent();
if (null != year && year.trim().length() == 4
&& this.isNumber(year.trim())) {
this.appendKeyValue(this.NUTCH_VIDEO_YEAR,
year.trim());
}
}

}

li = this.getFirstDescendantWithTagPlusAttribute(element,
LI, ID, AREABOX);
if (null != li) {
Element a = this.getFirstChildNodeWithTagName(li, A);
if (null != a) {
String area = a.getTextContent();
if (null != area && area.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_AREA, area);
}
}
if (null != li.getAttribute(CLASS))
li.removeAttribute(CLASS);

}

li = this.getFirstDescendantWithTagPlusAttribute(element,
LI, CLASS, S_H);
if (null != li) {
Element a = this.getFirstChildNodeWithTagName(li, A);
if (null != a) {
String type = a.getTextContent();
if (null != type && type.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_TYPE, type);
}
}
}

Element p = this.getFirstDescendantWithTagPlusAttribute(
element, P, CLASS, INTRO);
if (null != p) {
// delete a firstly!!!
Element em = this.getFirstChildNodeWithTagName(p, "em");
if (null != em) {
p.removeChild(em);
}
String text = p.getTextContent();
if (null != text && text.length() > 0) {
this.appendKeyValue(NUTCH_VIDEO_FULL_DESCRIPTION,
text);
}
}

return;
}
}

// handle children
NodeList children = node.getChildNodes();
for (int i = 0; children != null && i < children.getLength(); i++) {
walk(children.item(i));
}

} else if (nodeType == Node.TEXT_NODE) {
return;
} else if (nodeType == Node.COMMENT_NODE) {
return;
} else {
LOG.info("xxx-type-not-parsed------" + node.getNodeName());
return;
}

}

public ParseResult filter(Content content, ParseResult parseResult,
HTMLMetaTags metaTags, DocumentFragment doc) {
_parseResult = parseResult;
_content = content;
LOG.info("begin**********HtmlParseFilterIQiyi************by IQiyi");
/*
* LOG.info("Content Information:");
* LOG.info("BaseUrl---"+content.getBaseUrl());
* LOG.info("ContentType---"+content.getContentType());
* LOG.info("url---"+content.getUrl());
* LOG.info("Content---"+content.getContent().toString());
* LOG.info("Metadata"+content.getMetadata().toString());
*/
if (null == content || null == parseResult || null == metaTags
|| null == doc) {
LOG.info("content|parseResult|metaTags|doc is null,so just return parseResult...");
return parseResult;
}
LOG.info("four params checked ok,handle next......");
String url = content.getUrl();
LOG.info("currenturl is ------" + url);
if (false == this.isValidUrl(url)) {
LOG.info("invalid url,just return raw parseResult...");
return parseResult;
}
LOG.info("video[source]---" + "valid url,iqiyistatics...");

this.appendKeyValue(NUTCH_VIDEO_URL, url);

/*
* Metadata metadata = metaTags.getGeneralTags(); if (null != metadata)
* { this.appendKeyValue(NUTCH_VIDEO_KEYWORD, metadata.get(KEYWORDS));
* this.appendKeyValue(NUTCH_VIDEO_TITLE, metadata.get(TITLE));
* this.appendKeyValue(NUTCH_VIDEO_FULL_DESCRIPTION,
* metadata.get(DESCRIPTION)); }
*/

walk((Node) doc);

LOG.info("end**************************************end");
return parseResult;
}

public Configuration getConf() {
return conf;
}

public void setConf(Configuration conf) {
this.conf = conf;
}

}