Lucene自定义分词:合并IK分词+二元分词

索引命中效果不佳，采用IK分词与二元分词的效果都不是特别好，于是设计了新的自定义分词器，先将句子用IK分词分开，再对长度超过3的词进行二元分词。
以下是分词器的实现效果图。

实现思路先建立IK分词器，在通过第一层filter将IK分的词截留，长度大于等于3的词置入CJK分词器进行处理，然后得到的结果送入第二层filter中进行去重。因为在上一层中会出现大量重复词.下面贴代码。

package com.huang.analyzer;import java.io.BufferedReader;import java.io.IOException;import java.io.Reader;import java.io.StringReader;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.cjk.CJKAnalyzer;import org.apache.lucene.analysis.cjk.CJKBigramFilter;import org.apache.lucene.analysis.cjk.CJKTokenizer;import org.apache.lucene.analysis.cjk.CJKWidthFilter;import org.apache.lucene.analysis.core.LowerCaseFilter;import org.apache.lucene.analysis.core.LowerCaseTokenizer;import org.apache.lucene.analysis.core.UpperCaseFilter;import org.apache.lucene.analysis.standard.StandardFilter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.search.Filter;import org.apache.lucene.util.Version;import org.wltea.analyzer.lucene.IKAnalyzer;import org.wltea.analyzer.lucene.IKTokenizer;public class MyAnalyzer extends Analyzer {    private static final String text = "今天是个好日子";    //private static final String text = "How are you";    @Override    protected TokenStreamComponents createComponents(String fieldname, Reader in) {        // TODO Auto-generated method stub//      Tokenizer token = new LowerCaseTokenizer(in); //      TokenStreamComponents components = new TokenStreamComponents(token);//      TokenStream ts = components.getTokenStream();        //Tokenizer tokenizer = new IKBinaryTokenizer(in);        //Tokenizer tokenizer = new CJKTokenizer(in);        //TokenStream filter = new UpperCaseFilter(tokenizer);        //TokenStream filter = new CJKWidthFilter(tokenizer);        //Tokenizer tokenizer = new IkCjkTokenizer(in);        Tokenizer tokenizer = new IKTokenizer(in, false);        TokenStream normsFilter = new NormsFilter(tokenizer);        TokenStream distinctFilter = new DistinctFilter(normsFilter);        TokenStreamComponents components = new TokenStreamComponents(tokenizer, distinctFilter);        return components;    }    public static void main(String[] args) {        // TODO Auto-generated method stub        MyAnalyzer myAnalyzer = null;        myAnalyzer = new MyAnalyzer();                  StringReader reader = new StringReader(text);        //开始分词        try {            TokenStream ts = myAnalyzer.tokenStream("", reader);            ts.addAttribute(CharTermAttribute.class);            ts.reset();            int i = 0;            while(ts.incrementToken()){                CharTermAttribute cta = null;                cta = ts.getAttribute(CharTermAttribute.class);                OffsetAttribute offsetAtt = ts.getAttribute(OffsetAttribute.class);                System.out.println(cta.toString() + "(" + offsetAtt.startOffset() + "-" + offsetAtt.endOffset() + ")");            }            ts.end();            ts.close();            reader.close();            System.out.println();        }        catch (Exception e) {            // TODO Auto-generated catch block            e.printStackTrace();        }    }}

package com.huang.analyzer;import java.io.IOException;import java.io.StringReader;import java.nio.CharBuffer;import java.util.ArrayList;import java.util.Collection;import java.util.Iterator;import java.util.List;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.cjk.CJKAnalyzer;import org.apache.lucene.analysis.cjk.CJKTokenizer;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.analysis.tokenattributes.TypeAttribute;import org.apache.lucene.util.AttributeSource;public class NormsFilter extends TokenFilter{    /**中文词组长度过滤，默认超过2位长度的中文才转换拼音*/      private int minTermLength;    /**当前输入是否已输出*/      private boolean hasCurOut;      /**词元输入缓存*/      private char[] curTermBuffer;    /**词元输入长度*/      private int curTermLength;      private final CharTermAttribute termAtt = (CharTermAttribute) addAttribute(CharTermAttribute.class);      /**位置增量属性*/      private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);      private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);    /**cjk分词结果集*/    private Collection<String> terms;    /**结果集迭代器*/      private Iterator<String> termIte;      /**主词在字符串中的偏移量*/    private int firstStartOffset;    /**副词在主词中的偏移量*/    private int secondStartOffset;    private boolean norms;    protected NormsFilter(TokenStream input) {        super(input);    }    @Override    public boolean incrementToken() throws IOException {        // TODO Auto-generated method stub        //开始处理或上一词元已经处理完成        while (true) {            if (this.curTermBuffer == null) {                //获取下一词元输入                if (!this.input.incrementToken()) {                       // 没有后继词元输入，处理完成，返回false，结束上层调用                      return false;                   }                  this.curTermBuffer = ((char[]) this.termAtt.buffer().clone());                  this.curTermLength = this.termAtt.length();             }            //处理原输入词元            if (!hasCurOut && (this.termIte == null)) {                // 标记以保证下次循环不会输出                  this.hasCurOut = true;                   // 写入原输入词元                this.termAtt.copyBuffer(this.curTermBuffer, 0, this.curTermLength);                  this.posIncrAtt.setPositionIncrement(this.posIncrAtt.getPositionIncrement());                  this.typeAtt.setType("CN_WORD");                 if (curTermLength <= 2) {                    this.curTermBuffer = null;                    this.hasCurOut = false;                }                return true;              }            String text = this.termAtt.toString();            //norms为true表示在第二层中            if (norms == false) {                if (text.length() >= 3) {                    //主词相对句子的偏移量                    this.firstStartOffset = offsetAtt.startOffset();                    //开始第二层                    norms = true;                    //用容器来装cjk分好的二元分词                    Collection<String> coll = new ArrayList<String>();                    /**开始cjk分词*/                    StringReader reader = new StringReader(text);                    Tokenizer cjk = new CJKTokenizer(reader);                    cjk.reset();                    while (cjk.incrementToken()) {                        coll.add(cjk.getAttribute(CharTermAttribute.class).toString());                    }                    cjk.end();                    cjk.close();                    this.terms = coll;                    if (this.terms != null) {                          this.termIte = this.terms.iterator();                      }                  }else {                    this.termAtt.copyBuffer(curTermBuffer, 0, curTermLength);                    this.posIncrAtt.setPositionIncrement(this.posIncrAtt.getPositionIncrement());                      this.typeAtt.setType("CN_WORD");                    this.curTermBuffer = null;                      return true;                }            }            if (norms == true) {                  // 有拼音结果集且未处理完成                  while (this.termIte.hasNext()) {                    String pinyin = this.termIte.next();                      this.termAtt.copyBuffer(pinyin.toCharArray(), 0, pinyin.length());                      this.posIncrAtt.setPositionIncrement(this.posIncrAtt.getPositionIncrement());                      this.typeAtt.setType("CN_WORD");                    this.offsetAtt.setOffset(this.firstStartOffset, this.firstStartOffset + pinyin.length());                    this.firstStartOffset++;                    if (!this.termIte.hasNext()) {                        // 没有中文或转换拼音失败，不用处理，                          // 清理缓存，下次取新词元                          this.curTermBuffer = null;                          this.termIte = null;                          this.hasCurOut = false;                        this.firstStartOffset = 0;                        norms = false;                    }                    return true;                  }              }        }    }}

package com.huang.analyzer;import java.io.IOException;import java.util.Collection;import java.util.LinkedList;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.analysis.tokenattributes.TypeAttribute;public class DistinctFilter extends TokenFilter{    private Collection<String> terms;    /**词元输入缓存*/      private char[] curTermBuffer;    /**词元输入长度*/      private int curTermLength;      /**词元文字属性*/    private final CharTermAttribute termAtt = (CharTermAttribute) addAttribute(CharTermAttribute.class);      /**位置增量属性*/      private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);      /**词元类型属性*/    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);    protected DistinctFilter(TokenStream input) {        super(input);        // TODO Auto-generated constructor stub        this.terms = new LinkedList<String>();    }    @Override    public boolean incrementToken() throws IOException {        // TODO Auto-generated method stub        while (true) {            if (this.curTermBuffer == null) {                //获取下一词元输入                if (!this.input.incrementToken()) {                       // 没有后继词元输入，处理完成，返回false，结束上层调用                      return false;                   }                  this.curTermBuffer = ((char[]) this.termAtt.buffer().clone());                  this.curTermLength = this.termAtt.length();             }            String text = this.termAtt.toString();            if (terms.contains(text)) {                //this.curTermBuffer = "重复".toCharArray();                this.curTermBuffer = null;                continue;            }else {                terms.add(text);            }            /***/            this.termAtt.copyBuffer(this.curTermBuffer, 0, this.curTermLength);              this.posIncrAtt.setPositionIncrement(this.posIncrAtt.getPositionIncrement());              this.typeAtt.setType("CN_WORD");             /***/            this.curTermBuffer = null;            return true;        }    }    @Override    public void reset() throws IOException {        // TODO Auto-generated method stub        super.reset();        this.curTermBuffer = null;          terms.clear();    }}