JAVA基于Lucene实现建立索引以及检索

来源：互联网发布：七十二变软件编辑：程序博客网时间：2024/05/19 10:35

Step 1 利用Lucene建立索引

数据库中存放了数据集，对应上万个TXT的文件，表FILE含有两个字段，分别为path和content，依次保存文件的路径和内容。这里，利用Lucene对这些数据建立索引。

package pretreatment;import java.io.File;import java.io.IOException;import java.sql.Connection;import java.sql.DriverManager;import java.sql.SQLException;import java.util.Date;import java.util.List;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.index.IndexWriter;/** * BuildIndex用于创建索引，数据由数据库读取，创建的索引目录为INDEX_PATH * @author Tony * */public class BuildIndex{//索引存储路径private String INDEX_PATH = "D://index";public void createIndex(){try{//起始时间Date start = new Date();//创建分析器StandardAnalyzer standardAnalyzer = new StandardAnalyzer();//创建IndexWriterIndexWriter indexWriter = new IndexWriter(INDEX_PATH, standardAnalyzer, true); //参数设置indexWriter.setMergeFactor(100);indexWriter.setMaxBufferedDocs(100); indexWriter.setMaxFieldLength(1000); //从数据库获得数据对象Connection conn = BuildIndex.creatConn();int st = 0; int en = 49;int num = 1;while(en<=15900){List<FileInfo> fileList = FileInfoController.getFileInfo(st,en,conn);st = en+1;en = st+49;for (int i = 0; i < fileList.size(); i++) { FileInfo fInfo = (FileInfo) fileList.get(i);//System.out.println(fInfo.getContent());Document doc = new Document(); Field field = new Field("fileName",fInfo.getFileName(),Field.Store.YES,Field.Index.ANALYZED);doc.add(field);field = new Field("content",fInfo.getContent(),Field.Store.COMPRESS,Field.Index.ANALYZED,Field.TermVector.WITH_POSITIONS_OFFSETS);doc.add(field);indexWriter.addDocument(doc); System.out.println("****已完成对**"+fInfo.getFileName()+"**文件的分析****"+"ID:"+num);num++;} //fileList = null;//System.gc();}//执行优化indexWriter.optimize(); //关闭indexWriterindexWriter.close(); //关闭数据库连接conn.close();System.out.println("****Connection Successful Shut Down****");//结束时间Date end = new Date();System.out.println("****索引创建完毕，共耗时: " + (end.getTime() - start.getTime()) + "毫秒"); }catch(Exception e){System.out.println(e.toString());System.out.println("Unexpected Error Occoured ! Please restart the program later "); }}//建立数据库连接private static Connection creatConn(){Connection conn = null;try {Class.forName("com.mysql.jdbc.Driver").newInstance();} catch (InstantiationException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IllegalAccessException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (ClassNotFoundException e) {// TODO Auto-generated catch blocke.printStackTrace();}try {conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/txtsource?useUnicode=true&characterEncoding=utf-8", "root", "root");System.out.println("*****Set up Connection Successed!******");} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}return conn;}public static void main(String[] args){BuildIndex b = new BuildIndex();b.createIndex();}}

Step 2 查询索引

对于已经建立好的索引，下一步便是查询，这里，将查询语句写在ACTION中，然后将结果返回JSP进行显示。

package action;import java.io.IOException;import java.io.OutputStreamWriter;import java.io.PrintWriter;import java.io.StringReader;import java.util.ArrayList;import java.util.Date;import java.util.List;import javax.servlet.http.HttpServletRequest;import javax.servlet.http.HttpServletResponse;import javax.servlet.http.HttpSession;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.queryParser.ParseException;import org.apache.lucene.queryParser.QueryParser;import org.apache.lucene.search.Hits;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.highlight.Highlighter;import org.apache.lucene.search.highlight.QueryScorer;import org.apache.lucene.search.highlight.SimpleFragmenter;import org.apache.lucene.search.highlight.SimpleHTMLFormatter;import org.apache.struts2.ServletActionContext;import javaBean.ResultInfoBean;import com.opensymphony.xwork2.ActionContext;import com.opensymphony.xwork2.ActionSupport;public class SearchAction extends ActionSupport{/** * 接受查询输入 */private static final long serialVersionUID = 1L;private String keyWord;private int maxResults;private String path;private String content;private long costTime;private int resultSize;private String INDEX_PATH = "D://index";private StandardAnalyzer analyzer = new StandardAnalyzer();List<ResultInfoBean> lfi = new ArrayList<ResultInfoBean>();public List<ResultInfoBean> getLfi() {return lfi;}public void setLfi(List<ResultInfoBean> lfi) {this.lfi = lfi;}public long getCostTime() {return costTime;}public void setCostTime(long costTime) {this.costTime = costTime;}public int getResultSize() {return resultSize;}public void setResultSize(int resultSize) {this.resultSize = resultSize;}public String getPath() {return path;}public void setPath(String path) {this.path = path;}public String getContent() {return content;}public void setContent(String content) {this.content = content;}public String getKeyWord() {return keyWord;}public void setKeyWord(String keyWord) {this.keyWord = keyWord;}public int getMaxResults() {return maxResults;}public void setMaxResults(int maxResults) {this.maxResults = maxResults;}public String execute() throws Exception {// TODO Auto-generated method stubtry {//开始时间Date start = new Date();//BeginIndexSearcher searcher = new IndexSearcher(INDEX_PATH);QueryParser queryParse = new QueryParser("content", analyzer);Query query = queryParse.parse(keyWord); Hits hits = searcher.search(query); //高亮显示设置 //SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("",""); SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("",""); Highlighter highlighter = new Highlighter(simpleHTMLFormatter,new QueryScorer(query)); highlighter.setTextFragmenter(new SimpleFragmenter(50));//这个100是指定关键字字符串的context的长度，你可以自己设定，因为不可能返回整篇正文内容boolean bool = true; if(hits.length()==0){ bool = false;String ms = "没有找到相关的记录！";}else{for(int i=0;i<hits.length();i++){Document doc = hits.doc(i);//System.out.println("查询结果:"+doc.get("fileName"));//System.out.println(doc.get("content"));//高亮出显示 TokenStream tokenStream =analyzer.tokenStream("content", new StringReader(doc.get("content"))); content = highlighter.getBestFragment(tokenStream,doc.get("content")); //System.out.println(content); String path = doc.get("fileName");ResultInfoBean ri = new ResultInfoBean();ri.setPath(path);ri.setContent(content);lfi.add(ri);}}//结束时间Date end = new Date();long timeOfSearch = end.getTime() - start.getTime();System.out.println("搜索耗时:"+timeOfSearch); costTime=timeOfSearch;//共耗时resultSize = hits.length();//共找到条记录// HttpServletRequest request = ServletActionContext.getRequest(); request.setAttribute("sl", 0); request.setAttribute("numPerPage", maxResults); HttpSession session = request.getSession(); session.setAttribute("lfi", lfi); } catch (CorruptIndexException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}return SUCCESS;}}

Step 3 显示与分页

对于分页机制，我选择的是将用户查询的结果保存在SESSION中，每次在SESSION中取相应大小的结果，这样做的结果是用户在点击查询的时候，速度相对比较慢，但分页时就不需要重复查询。当然了，个人认为这是个比较stupid的方法，但一时也没想到更好的方法，只好将就了……

<%@ page language="java" import ="java.util.*" import="javaBean.*;" pageEncoding="UTF-8"%><%@ taglib prefix="s" uri="/struts-tags" %><!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"><html> <head> <title>Search Result</title> <link rel="stylesheet" type="text/css" href="css/styles.css" mce_href="css/styles.css"> <mce:style type="text/css"></mce:style><style type="text/css" mce_bogus="1"> .highlight{ color: #C71585; font-weight: bold;} </style> </head> <s:include value="head.jsp"/> <body> <HR style="FILTER: alpha(opacity=100,finishopacity=0,style=1)" mce_style="FILTER: alpha(opacity=100,finishopacity=0,style=1)" width="100%" color=#0000CD SIZE=1> <% //初始化全局变量 int totalNum = 0; //总共记录条数 int numPerPage = 0; //每页显示记录条数 int maxPageNum = 0; //本次搜索最大页数 int curPage = 1; //当前页 int startLocation = 0; //起始位置 List<ResultInfoBean> searchresult = new ArrayList<ResultInfoBean>(); searchresult = (List<ResultInfoBean>) session.getAttribute("lfi"); startLocation = (Integer)request.getAttribute("sl"); numPerPage = (Integer)request.getAttribute("numPerPage"); totalNum = searchresult.size(); maxPageNum=((totalNum%numPerPage) == 0)?totalNum/numPerPage:(totalNum/numPerPage+1); curPage=startLocation/numPerPage + 1; %> <% if(curPage<maxPageNum){ System.out.println("startLocation="+startLocation); for(int i=startLocation; i<startLocation+numPerPage; i++){ ResultInfoBean result = (ResultInfoBean) searchresult.get(i); out.print(""+result.getPath()+""); out.println(""+result.getContent()); } }else{ System.out.println("totalNum="+totalNum); for(int i=startLocation; i<totalNum; i++){ ResultInfoBean result = (ResultInfoBean) searchresult.get(i); out.print(""+result.getPath()+""); out.println(""+result.getContent()); } } %> <% if (curPage > 1 ) { out.println("<a href="PagingControl.action?startLocation="+(startLocation-numPerPage)); out.println("" mce_href="PagingControl.action?startLocation="+(startLocation-numPerPage)); out.println("">上一页</a>"); } if(maxPageNum > 10) { int totalNumShow = curPage + 9; int startLocationPage = 0; if( totalNumShow > maxPageNum) { totalNumShow = maxPageNum; } if( maxPageNum > 11 && curPage > 11) { startLocationPage = curPage - 11; } for(int i=startLocationPage;i<totalNumShow;i++) { if(i+1 == curPage) { %> [<%=i+1%>] <% } else { %><a href="PagingControl.action?startLocation=<%=i*numPerPage%>"> [<%=i+1 %>] </a><% } } } else { for( int i=0;i<maxPageNum;i++) { if( i+1 == curPage) { %> [<%=i+1 %>] <% } else { %><a href="PagingControl.action?startLocation=<%=i*numPerPage%>"> [<%=i+1 %>] </a><% } } } if ( curPage < maxPageNum) { out.println("<a href="PagingControl.action?startLocation="+(startLocation+numPerPage)); out.println("" mce_href="PagingControl.action?startLocation="+(startLocation+numPerPage)); out.println("">下一页</a>"); } %> </body></html>

这个程序的效果就跟Google差不多，只不过查询只限于本地数据库中已存好的数据。