nutch-0.8在eclipse中运行

nutch-0.8在eclipse中运行

运行环境：

1. jdk 1.5
2. eclipse 3.2
3. nutch-0.8已解压制D:/ (注：此目录是随意的)

---

nutch 0.8缺少的两个jar包：

1. 下载---关于parse-mp3的jid3lib-0.5.1.jar
   
2. 下载---关于parse-rtf的rtf-parser.jar

分别拷贝至

1. D:/nutch-0.8.1/src/plugin/parse-mp3/lib
2. D:/nutch-0.8.1/src/plugin/parse-rtf/lib

---

配置文件的更改：
主要更改的配置文件为如下三个：

1. nutch-default.xml
   更改plugin.folders的value为D:/nutch-0.8.1/src/plugin
   更改http.agent.name的value为godric(注：此处随意‘)
2. 在D:/nutch-0.8.1下添加文件夹urls,在urls目录下新建文本文档nutch.txt(注：文件名随意)，内容为你要爬取得网站url，例如：http://www.iscas.ac.cn
3. crawl-urlfilter.txt
   修改为如下所示：
   # accept hosts in MY.DOMAIN.NAME
   +^http://www.iscas.ac.cn/
   

---

项目文件：

1. .classpath 其内容为：
   <?xml version="1.0" encoding="UTF-8"?>
   <classpath>
       <classpathentry kind="src" path="conf"/>
       <classpathentry kind="src" path="src/java"/>
       <classpathentry kind="src" path="src/test"/>
       <classpathentry kind="src" path="src/plugin/creativecommons/src/java"/>
       <classpathentry kind="src" path="src/plugin/summary-lucene/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-rtf/src/java"/>
       <classpathentry kind="src" path="src/plugin/query-url/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-rss/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-msword/src/java"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-suffix/src/test"/>
       <classpathentry kind="src" path="src/plugin/protocol-http/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-pdf/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-zip/src/java"/>
       <classpathentry kind="src" path="src/plugin/summary-basic/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-rtf/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-swf/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-oo/src/java"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-regex/src/test"/>
       <classpathentry kind="src" path="src/plugin/protocol-httpclient/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-msword/src/test"/>
       <classpathentry kind="src" path="src/plugin/subcollection/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-oo/src/test"/>
       <classpathentry kind="src" path="src/plugin/protocol-http/src/test"/>
       <classpathentry kind="src" path="src/plugin/query-url/src/test"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-prefix/src/java"/>
       <classpathentry kind="src" path="src/plugin/protocol-ftp/src/java"/>
       <classpathentry kind="src" path="src/plugin/query-site/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-rss/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-swf/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-zip/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-text/src/java"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-suffix/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-pdf/src/java"/>
       <classpathentry kind="src" path="src/plugin/protocol-file/src/java"/>
       <classpathentry kind="src" path="src/plugin/query-more/src/java"/>
       <classpathentry kind="src" path="src/plugin/scoring-opic/src/java"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-automaton/src/test"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-automaton/src/java"/>
       <classpathentry kind="src" path="src/plugin/query-basic/src/java"/>
       <classpathentry kind="src" path="src/plugin/urlfilter-regex/src/java"/>
       <classpathentry kind="src" path="src/plugin/subcollection/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-ext/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-mp3/src/test"/>
       <classpathentry kind="src" path="src/plugin/ontology/src/test"/>
       <classpathentry kind="src" path="src/plugin/creativecommons/src/test"/>
       <classpathentry kind="src" path="src/plugin/lib-parsems/src/java"/>
       <classpathentry kind="src" path="src/plugin/ontology/src/java"/>
       <classpathentry kind="src" path="src/plugin/index-basic/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-js/src/java"/>
       <classpathentry kind="src" path="src/plugin/index-more/src/java"/>
       <classpathentry kind="src" path="src/plugin/lib-regex-filter/src/test"/>
       <classpathentry kind="src" path="src/plugin/analysis-de/src/java"/>
       <classpathentry kind="src" path="src/plugin/microformats-reltag/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-mspowerpoint/src/java"/>
       <classpathentry kind="src" path="src/plugin/lib-http/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-msexcel/src/test"/>
       <classpathentry kind="src" path="src/plugin/languageidentifier/src/java"/>
       <classpathentry kind="src" path="src/plugin/lib-regex-filter/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-html/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-html/src/java"/>
       <classpathentry kind="src" path="src/plugin/clustering-carrot2/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-ext/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-mspowerpoint/src/test"/>
       <classpathentry kind="src" path="src/plugin/parse-mp3/src/java"/>
       <classpathentry kind="src" path="src/plugin/parse-msexcel/src/java"/>
       <classpathentry kind="src" path="src/plugin/lib-http/src/test"/>
       <classpathentry kind="src" path="src/plugin/analysis-fr/src/java"/>
       <classpathentry kind="src" path="src/plugin/clustering-carrot2/src/java"/>
       <classpathentry kind="src" path="src/plugin/languageidentifier/src/test"/>
       <classpathentry kind="lib" path="lib/commons-cli-2.0-SNAPSHOT.jar"/>
       <classpathentry kind="lib" path="lib/commons-lang-2.1.jar"/>
       <classpathentry kind="lib" path="lib/commons-logging-1.0.4.jar"/>
       <classpathentry kind="lib" path="lib/commons-logging-api-1.0.4.jar"/>
       <classpathentry kind="lib" path="lib/concurrent-1.3.4.jar"/>
       <classpathentry kind="lib" path="lib/hadoop-0.4.0-patched.jar"/>
       <classpathentry kind="lib" path="lib/jakarta-oro-2.0.7.jar"/>
       <classpathentry kind="lib" path="lib/jetty-5.1.4.jar"/>
       <classpathentry kind="lib" path="lib/junit-3.8.1.jar"/>
       <classpathentry kind="lib" path="lib/log4j-1.2.13.jar"/>
       <classpathentry kind="lib" path="lib/lucene-core-1.9.1.jar"/>
       <classpathentry kind="lib" path="lib/lucene-misc-1.9.1.jar"/>
       <classpathentry kind="lib" path="lib/servlet-api.jar"/>
       <classpathentry kind="lib" path="lib/taglibs-i18n.jar"/>
       <classpathentry kind="lib" path="lib/xerces-2_6_2.jar"/>
       <classpathentry kind="lib" path="lib/xerces-2_6_2-apis.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/carrot2-filter-lingo.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/carrot2-local-core.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/carrot2-snowball-stemmers.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/carrot2-util-common.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/carrot2-util-tokenizer.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/commons-collections-3.1-patched.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/commons-pool-1.1.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/Jama-1.0.1-patched.jar"/>
       <classpathentry kind="lib" path="src/plugin/clustering-carrot2/lib/violinstrings-1.0.2.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-commons-httpclient/lib/commons-httpclient-3.0.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-jakarta-poi/lib/poi-3.0-alpha1-20050704.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-jakarta-poi/lib/poi-scratchpad-3.0-alpha1-20050704.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-log4j/lib/log4j-1.2.11.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-1.9-rc1-dev.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-nekohtml/lib/nekohtml-0.9.4.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-xml/lib/jaxen-core.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-xml/lib/jaxen-jdom.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-xml/lib/jdom.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-xml/lib/saxpath.jar"/>
       <classpathentry kind="lib" path="src/plugin/lib-xml/lib/xercesImpl.jar"/>
       <classpathentry kind="lib" path="src/plugin/ontology/lib/commons-logging-1.0.3.jar"/>
       <classpathentry kind="lib" path="src/plugin/ontology/lib/icu4j_2_6_1.jar"/>
       <classpathentry kind="lib" path="src/plugin/ontology/lib/jena-2.1.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-html/lib/tagsoup-1.0rc3.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-pdf/lib/PDFBox-0.7.2-log4j.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-rss/lib/commons-feedparser-0.6-fork.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-rss/lib/xmlrpc-1.2.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-swf/lib/javaswf.jar"/>
       <classpathentry kind="lib" path="src/plugin/protocol-ftp/lib/commons-net-1.2.0-dev.jar"/>
       <classpathentry kind="lib" path="src/plugin/protocol-httpclient/lib/commons-codec.jar"/>
       <classpathentry kind="lib" path="src/plugin/summary-lucene/lib/lucene-highlighter-2.0-rc1-dev.jar"/>
       <classpathentry kind="lib" path="src/plugin/urlfilter-automaton/lib/automaton.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-mp3/lib/jid3lib-0.5.1.jar"/>
       <classpathentry kind="lib" path="src/plugin/parse-rtf/lib/rtf-parser.jar"/>
       <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
       <classpathentry kind="output" path="tmp_build"/>
   </classpath>
   
   
   
2. .project 其内容为：
   <?xml version="1.0" encoding="UTF-8"?>
   <projectDescription>
       <name>nutch-0.8</name>
       <comment></comment>
       <projects>
       </projects>
       <buildSpec>
           <buildCommand>
               <name>org.eclipse.jdt.core.javabuilder</name>
               <arguments>
               </arguments>
           </buildCommand>
       </buildSpec>
       <natures>
           <nature>org.eclipse.jdt.core.javanature</nature>
       </natures>
   </projectDescription>

---

导入eclipse：

1. File->Import选择目录D:/nutch-0.8.1
2. Run->Run...
   配置如下：
   
   
   Main class:  org.apache.nutch.crawl.Crawl
   Program arguments: urls -dir crawl.iscas -depth 2 -topN 50
   VM arguments: -Dhadoop.log.dir=logs -Dhadoop.log.file=hadoop.log

---

不知您是否运行，如果还没有，请email我：[e-mail]