全文检索Lucene（三）--中文分词与高亮显示

一、中文分词smartcn

二、检索结果高亮显示实现

       首先，创建maven项目，添加相关依赖。

<dependencies>  <dependency><groupId>org.apache.lucene</groupId><artifactId>lucene-core</artifactId><version>5.3.1</version></dependency><dependency><groupId>org.apache.lucene</groupId><artifactId>lucene-queryparser</artifactId><version>5.3.1</version></dependency><dependency><groupId>org.apache.lucene</groupId><artifactId>lucene-analyzers-common</artifactId><version>5.3.1</version></dependency><dependency><groupId>org.apache.lucene</groupId><artifactId>lucene-highlighter</artifactId><version>5.3.1</version></dependency><dependency><groupId>org.apache.lucene</groupId><artifactId>lucene-analyzers-smartcn</artifactId><version>5.3.1</version></dependency><dependency><groupId>junit</groupId><artifactId>junit</artifactId><version>4.12</version></dependency>  </dependencies>

       其次，建立索引，自己在D盘创建lucene6文件夹

package com.feiyang.lucene3;import java.nio.file.Paths;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.IntField;import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FSDirectory;public class Indexer {private Integer ids[]={1,2,3};private String citys[]={"青岛","南京","上海"};private String descs[]={"青岛是一个美丽的城市。","南京是一个有文化的城市。南京是一个文化的城市南京，简称宁，是江苏省会，地处中国东部地区，长江下游，濒江近海。全市下辖11个区，总面积6597平方公里，2013年建成区面积752.83平方公里，常住人口818.78万，其中城镇人口659.1万人。[1-4] “江南佳丽地，金陵帝王州”，南京拥有着6000多年文明史、近2600年建城史和近500年的建都史，是中国四大古都之一，有“六朝古都”、“十朝都会”之称，是中华文明的重要发祥地，历史上曾数次庇佑华夏之正朔，长期是中国南方的政治、经济、文化中心，拥有厚重的文化底蕴和丰富的历史遗存。[5-7] 南京是国家重要的科教中心，自古以来就是一座崇文重教的城市，有“天下文枢”、“东南第一学”的美誉。截至2013年，南京有高等院校75所，其中211高校8所，仅次于北京上海；国家重点实验室25所、国家重点学科169个、两院院士83人，均居中国第三。[8-10] 。","上海是一个繁华的城市。"};private Directory dir;/** * 获取IndexWriter实例 * @return * @throws Exception */private IndexWriter getWriter()throws Exception{//Analyzer analyzer=new StandardAnalyzer(); // 标准分词器SmartChineseAnalyzer analyzer=new SmartChineseAnalyzer();IndexWriterConfig iwc=new IndexWriterConfig(analyzer);IndexWriter writer=new IndexWriter(dir, iwc);return writer;}/** * 生成索引 * @param indexDir * @throws Exception */private void index(String indexDir)throws Exception{dir=FSDirectory.open(Paths.get(indexDir));IndexWriter writer=getWriter();for(int i=0;i<ids.length;i++){Document doc=new Document();doc.add(new IntField("id", ids[i], Field.Store.YES));doc.add(new StringField("city",citys[i],Field.Store.YES));doc.add(new TextField("desc", descs[i], Field.Store.YES));writer.addDocument(doc); // 添加文档}writer.close();}public static void main(String[] args) throws Exception {new Indexer().index("D:\\lucene6");}}

    

   最后，查询关键字，并高亮显示我们查询的关键字

package com.feiyang.lucene3;import java.io.StringReader;import java.nio.file.Paths;import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.queryparser.classic.QueryParser;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.ScoreDoc;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.highlight.Fragmenter;import org.apache.lucene.search.highlight.Highlighter;import org.apache.lucene.search.highlight.QueryScorer;import org.apache.lucene.search.highlight.SimpleHTMLFormatter;import org.apache.lucene.search.highlight.SimpleSpanFragmenter;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FSDirectory;public class Searcher {public static void search(String indexDir,String q)throws Exception{Directory dir=FSDirectory.open(Paths.get(indexDir));IndexReader reader=DirectoryReader.open(dir);IndexSearcher is=new IndexSearcher(reader);//中文分词器SmartChineseAnalyzer analyzer=new SmartChineseAnalyzer();QueryParser parser=new QueryParser("desc", analyzer);//创建查询规则Query query=parser.parse(q);long start=System.currentTimeMillis();//获取命中文档TopDocs hits=is.search(query, 10);long end=System.currentTimeMillis();System.out.println("匹配 "+q+" ，总共花费"+(end-start)+"毫秒"+"查询到"+hits.totalHits+"个记录");//创建QueryScoerQueryScorer scorer=new QueryScorer(query);//获得Fragmenter fragmenter=new SimpleSpanFragmenter(scorer);//自定义高亮显示格式SimpleHTMLFormatter simpleHTMLFormatter=new SimpleHTMLFormatter("<b><font color='red'>","</font></b>");Highlighter highlighter=new Highlighter(simpleHTMLFormatter, scorer);highlighter.setTextFragmenter(fragmenter);for(ScoreDoc scoreDoc:hits.scoreDocs){Document doc=is.doc(scoreDoc.doc);System.out.println(doc.get("city"));System.out.println(doc.get("desc"));String desc=doc.get("desc");if(desc!=null){TokenStream tokenStream=analyzer.tokenStream("desc", new StringReader(desc));//获取最高分的文本片段，即摘要System.out.println(highlighter.getBestFragment(tokenStream, desc));}}reader.close();}public static void main(String[] args) {String indexDir="D:\\lucene6";String q="南京文明";try {search(indexDir,q);} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}}}