开发基于 Nutch 的集群式搜索引擎

<property>

<name>http.agent.name</name>

<value>Nutch-hadoop</value>

<description>HTTP 'User-Agent' request header. MUST NOT be empty -

please set this to a single word uniquely related to your organization.

</description>

</property>

<property>

<name>http.agent.description</name>

<value>bydenver</value>

<description>Further description of our bot- this text is used in

the User-Agent header.  It appears in parenthesis after the agent name.

</description>

</property>

<property>

<name>http.agent.url</name>

<value>myNutch.com</value>

<description>A URL to advertise in the User-Agent header.  This will

appear in parenthesis after the agent name. Custom dictates that this

should be a URL of a page explaining the purpose and behavior of this

crawler.

</description>

</property>

<property>

<name>http.agent.email</name>

<value>wangfp@cn.ibm.com</value>

<description>An email address to advertise in the HTTP 'From' request

header and User-Agent header. A good practice is to mangle this

address (e.g. 'info at example dot com') to avoid spamming.

</description>

</property>

<property>

<name>fs.default.name</name>

<value>9.181.87.172:9000</value>

<description>

The name of the default file system. </description>

</property>

<property>

<name>mapred.job.tracker</name>

<value>9.181.87.172:9001</value>

<description>

The host and port that the MapReduce job tracker runs at.

</description>

</property>

<property>

<name>mapred.map.tasks</name>

<value>2</value>

<description>

define mapred.map tasks to be number of slave hosts

</description>

</property>

<property>

<name>mapred.reduce.tasks</name>

<value>2</value>

<description>

define mapred.reduce tasks to be number of slave hosts

</description>

</property>

<property>

<name>dfs.name.dir</name>

<value>/workspace/filesystem/name</value>

</property>

<property>

<name>dfs.data.dir</name>

<value>/workspace/filesystem/data</value>

</property>

<property>

<name>mapred.system.dir</name>

<value>/workspace/filesystem/mapreduce/system</value>

</property>

<property>

<name>mapred.local.dir</name>

<value>/workspace/filesystem/mapreduce/local</value>

</property>

<property>

<name>dfs.replication</name>

<value>2</value>

</property>

9.181.87.176

cd /workspace/Nutch-0.8.1

mkdir urls

echo http://myNutch.com/targetWebApp > urls/urllist.txt

conf/crawl-urlfilter.txt

+^http://([a-z0-9]*/.)*myNutch.com/

bin/hadoop dfs -put urls urls

bin/hadoop dfs namenode –format

bin/Nutch crawl urls -dir ./crawl -depth 3 -topN 10

package org.myNutch;

import java.io.IOException;

import java.io.*;

import java.util.*;

import org.apache.hadoop.conf.Configuration;

import org.apache.Nutch.searcher.*;

import org.apache.Nutch.util.*;

public class NutchApp {

/** For debugging. */

public static void main(String[] args) throws Exception {

String usage = "NutchBean query";

if (args.length == 0) {

System.err.println(usage);

System.exit(-1);

}

Configuration conf = NutchConfiguration.create();

NutchBean bean = new NutchBean(conf);

Query query = Query.parse(args[0], conf);

Hits hits = bean.search(query, 10);

System.out.println("Total hits: " + hits.getTotal());

int length = (int)Math.min(hits.getTotal(), 10);

Hit[] show = hits.getHits(0, length);

HitDetails[] details = bean.getDetails(show);

Summary[] summaries = bean.getSummary(details, query);

for ( int i = 0; i <hits.getLength();i++){

System.out.println(" "+i+" "+ details[i] + "/n" + summaries[i]);

}

}

}

Javac -cp "Nutch-0.8.1.jar;hadoop-0.4.0-patched.jar" src/org/myNutch/NutchApp.java -d lib

cd lib

jar cvf NutchApp.jar org/myNutch/NutchApp.class

cd ../

bin/Nutch org.myNutch.NutchApp Nutch

Total hits: 3

0 20061104142342/http://myNutch.com/targetWebApp/two.html

... 8 release of Nutch is now available. This is ... first release of Nutch

1 20061104142342/http://myNutch.com/targetWebApp/one.html

... 1 release of Nutch is now available. This is ...

2 20061104142342/http://myNutch.com/targetWebApp/three.html

... 2 release of Nutch is now available. This is ...