利用Nutch实现分类搜索（二）（加入urlfilter plugin）

其实有了上篇文章的index plugin，Crawl已经可以正确地把网址根据不同的type分类索引，本文加入urlfilter plugin是为了更进一步，把Crawl抓取的网址限定在我们定义的type之类，而不是什么网页都抓取。

在src/plugin/目录下加入urlfilter-type目录，其中的文件结构可以参考urlfilter-regex，并作相应的修改。

加入TypeURLFilter.java文件，目录层次为urlfilter-type/src/java/com/zju/repu/urlfilter/type/TypeURLFilter.java，实现URLFilter接口。

package com.zju.repu.urlfilter.type;// JDK importsimport java.io.Reader;import java.io.IOException;import java.io.BufferedReader;import java.io.InputStreamReader;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;import java.util.HashMap;import java.util.List;import java.util.ArrayList;// Hadoop importsimport org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.hadoop.conf.Configuration;import org.apache.nutch.net.*;import org.apache.nutch.urlfilter.api.RegexRule;import org.apache.nutch.urlfilter.api.Rule;import org.apache.nutch.util.NutchConfiguration;public class TypeURLFilter implements URLFilter{ /** My logger */ private final static Log LOG = LogFactory.getLog (TypeURLFilter.class); // 该HashMap以网页类型作为key，以Rule数组作为Value HashMap<String, RegexRule[]> rules = new HashMap<String, RegexRule[]> (); /** The current configuration */ private Configuration conf; public TypeURLFilter() { } // Inherited Javadoc protected String getRulesFile (String ruleName) { // 从配置变量中获取规则文件 String file = conf.get ("urlfilter." + ruleName + ".file"); LOG.info ("getRulesFile:" + ruleName + ".file=" + file); return file; } // Inherited Javadoc protected RegexRule createRule (boolean sign, String regex) { return new Rule (sign, regex); } // 读取所有的规则类型，并以数组返回 protected String[] getRulesName () { String typeNames = conf.get ("urlfilter.type.includes"); LOG.debug ("getRulesName: " + typeNames); return typeNames.split ("//|"); } // 读取所有规则文件，并根据规则名称保存到HashMap中 protected void init () { String[] ruleNames = getRulesName (); for (int index = 0; index < ruleNames.length; index++) { try { String fileName = getRulesFile (ruleNames[index]); Reader reader = conf.getConfResourceAsReader (fileName); if (reader == null) { LOG.fatal ("Can't find resource: " + fileName); continue; } RegexRule[] temprules = readRulesFile (reader); rules.put (ruleNames[index], temprules); LOG.debug ("Add the urlfilter rule to the hashmap: " + fileName); } catch (Exception e) { LOG.error ("Add the urlfilter failed: " + ruleNames[index]); LOG.error (e.toString ()); continue; } } } // 判断一个Rule数组是否能接受该url，如果接受返回true，否则false protected boolean acceptUrl(RegexRule[] inRules, String url) { for (int i = 0; i < inRules.length; i++) { if (inRules[i].match (url)) { LOG.debug ("acceptUrl for: " + url); return inRules[i].accept (); } } return false; } // 这个函数是提供给Nutch的接口函数，判断是否接受一个url public synchronized String filter (String url) { // 针对每种网页类型，分别判断是否接受 for (String ruleName: rules.keySet ()) { if (acceptUrl(rules.get (ruleName),url)) { LOG.debug ("filter for: " + ruleName); return url; } } LOG.info ("filter reject for: " + url); return null; } public void setConf (Configuration conf) { this.conf = conf; init (); } public Configuration getConf () { return this.conf; } /** * Read the specified file of rules. * * @param reader * is a reader of regular expressions rules. * @return the corresponding {@RegexRule rules}. */ private RegexRule[] readRulesFile (Reader reader) throws IOException, IllegalArgumentException { BufferedReader in = new BufferedReader (reader); List inrules = new ArrayList (); String line; while ((line = in.readLine ()) != null) { if (line.length () == 0) { continue; } char first = line.charAt (0); boolean sign = false; switch (first) { case '+': sign = true; break; case '-': sign = false; break; case ' ': case '/n': case '#': // skip blank & comment lines continue; default: throw new IOException ("Invalid first character: " + line); } String regex = line.substring (1); if (LOG.isTraceEnabled ()) { LOG.trace ("Adding rule [" + regex + "]"); } RegexRule rule = createRule (sign, regex); inrules.add (rule); } return (RegexRule[]) inrules.toArray (new RegexRule[inrules.size ()]); }}  

 

加入build.xml，用到了lib-regex-filter库，所以要加入引用。

<?xml version="1.0"?><project name="urlfilter-type" default="jar-core"> <import file="../build-plugin.xml"/> <!-- Build compilation dependencies --> <target name="deps-jar"> <ant target="jar" inheritall="false" dir="../lib-regex-filter"/> <ant target="compile-test" inheritall="false" dir="../lib-regex-filter"/> </target> <!-- Add compilation dependencies to classpath --> <path id="plugin.deps"> <fileset dir="${nutch.root}/build"> <include name="**/lib-regex-filter/*.jar" /> </fileset> <pathelement location="${nutch.root}/build/lib-regex-filter/test"/> </path> <!-- Deploy Unit test dependencies --> <target name="deps-test"> <ant target="deploy" inheritall="false" dir="../lib-regex-filter"/> </target></project> 

加入plugin.xml，注意extention point是"org.apache.nutch.net.URLFilter"，是urlfilter的公共的接口。

<?xml version="1.0" encoding="UTF-8"?><plugin id="urlfilter-type" name="Web Type URL Filter" version="1.0.0" provider-name="repu.com"> <runtime> <library name="urlfilter-type.jar"> <export name="*"/> </library> </runtime> <requires> <import plugin="nutch-extensionpoints"/> <import plugin="lib-regex-filter"/> </requires> <extension id="com.zju.repu.urlfilter.type" name="Nutch Web Type URL Filter" point="org.apache.nutch.net.URLFilter"> <implementation id="TypeURLFilter" class="com.zju.repu.urlfilter.type.TypeURLFilter"/> </extension></plugin> 

 

另外还需要把这个urlfilter-type加入编译系统，修改src/plugin/build.xml，添加如下编译入口：

<target name="deploy"> <ant dir="urlfilter-type" target="deploy"/></target><target name="clean"> <ant dir="urlfilter-type" target="clean"/></target>

为了在让urlfilter-type能够读取到filter文件，还需要在nutch-site.xml中加入如下的配置：

<property> <name>urlfilter.news.file</name> <value>crawl-urltype-news.txt</value> <description>Name of file on CLASSPATH containing regular expressions used by urlfilter-repu(RepuURLFilter) plugin.</description> </property> <property> <name>urlfilter.shop.file</name> <value>crawl-urltype-shop.txt</value> <description>Name of file on CLASSPATH containing regular expressions used by urlfilter-repu(RepuURLFilter) plugin.</description> </property> <property> <name>urlfilter.blog.file</name> <value>crawl-urltype-blog.txt</value> <description>Name of file on CLASSPATH containing regular expressions used by urlfilter-repu(RepuURLFilter) plugin.</description> </property> <property> <name>urlfilter.disc.file</name> <value>crawl-urltype-disc.txt</value> <description>Name of file on CLASSPATH containing regular expressions used by urlfilter-repu(RepuURLFilter) plugin.</description> </property> <property> <name>urlfilter.qa.file</name> <value>crawl-urltype-qa.txt</value> <description>Name of file on CLASSPATH containing regular expressions used by urlfilter-repu(RepuURLFilter) plugin.</description> </property> 

至此，Crawl能够根据规则文件的设定，把网页抓取限定于定义的规则之内了。