Lucene分词器解析-代码篇

</pre><pre code_snippet_id="1596031" snippet_file_name="blog_20160303_2_418520" name="code" class="java"><pre name="code" class="java">Lucene版本：5.2.1java版本：1.8测试时间：2016-3-3 16:04:57显示分词效果

public static void displayToken(String str,Analyzer a) {try {TokenStream stream = a.tokenStream("content",new StringReader(str));//创建一个属性，这个属性会添加流中，随着这个TokenStream增加CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);stream.reset();while(stream.incrementToken()) {System.out.print("["+cta+"]");}stream.end();System.out.println();} catch (IOException e) {e.printStackTrace();}}

//显示所有token过程

<span style="white-space:pre"></span>public static void displayAllTokenInfo(String str,Analyzer a) {try {TokenStream stream = a.tokenStream("content",new StringReader(str));//位置增量的属性，存储语汇单元之间的距离PositionIncrementAttribute pia = stream.addAttribute(PositionIncrementAttribute.class);//每个语汇单元的位置偏移量OffsetAttribute oa = stream.addAttribute(OffsetAttribute.class);//存储每一个语汇单元的信息（分词单元信息）CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);//使用的分词器的类型信息TypeAttribute ta = stream.addAttribute(TypeAttribute.class);stream.reset();for(;stream.incrementToken();) {System.out.print(pia.getPositionIncrement()+":");System.out.print(cta+"["+oa.startOffset()+"-"+oa.endOffset()+"]-->"+ta.type()+"\n");}} catch (Exception e) {e.printStackTrace();}}

@Test//测试工具类

public void test01() {Analyzer a1 = new StandardAnalyzer();Analyzer a2 = new StopAnalyzer();Analyzer a3 = new SimpleAnalyzer();Analyzer a4 = new WhitespaceAnalyzer();String txt = "this is GS/MC " +"My email is [e-mail],My QQ is 65356435";AnalyzerUtils.displayToken(txt, a1);AnalyzerUtils.displayToken(txt, a2);AnalyzerUtils.displayToken(txt, a3);AnalyzerUtils.displayToken(txt, a4);}