Lucene九（分词）

Lucene3.5中有这四大类分词器：SimpleAnalyzer、StopAnalyzer、WhiltespaceAnalyzer、StandardAnalyzer。

分词流程：

Tokenizer：

TokenFilter：

TokenStream中保存的信息如下：

了解了分词流程和TokenStream之后，可以通过代码来查看分词信息：

package cn.liuys.lucene.util;
import java.io.IOException;
import java.io.StringReader;


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;




public class AnalyzerUtil {

/**
* @param str
* @param a
* 显示分词详细信息
*/
public static void displayAllTokenInfo(String str,Analyzer a){
TokenStream stream = a.tokenStream("content", new StringReader(str));
//位置增量
PositionIncrementAttribute pia = stream.addAttribute(PositionIncrementAttribute.class);
//偏移量
OffsetAttribute oa = stream.addAttribute(OffsetAttribute.class);
//词汇单元
CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);
//类型
TypeAttribute ta = stream.addAttribute(TypeAttribute.class);
try {
for(;stream.incrementToken();){
System.out.print(pia.getPositionIncrement()+":");
System.out.print(cta+"["+oa.startOffset()+"-"+oa.endOffset()+"]--->"+ta.type()+"\n");
}
} catch (IOException e) {
e.printStackTrace();
}
}

}

package cn.liuys.lucene.test;


import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.Version;
import org.junit.Test;


import cn.liuys.lucene.util.AnalyzerUtil;


public class TestAnalyzer {

@Test
public void test03(){
Analyzer a1 = new StandardAnalyzer(Version.LUCENE_35);
Analyzer a2 = new StopAnalyzer(Version.LUCENE_35);
Analyzer a3 = new SimpleAnalyzer(Version.LUCENE_35);
Analyzer a4 = new WhitespaceAnalyzer(Version.LUCENE_35);

String str = "how are you thank you";
AnalyzerUtil.displayAllTokenInfo(str, a1);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a2);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a3);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a4);
}
}