代码测试

import org.apache.lucene.index.IndexWriter;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.analysis.standard.StandardAnalyzer;import java.io.BufferedReader;import java.io.FileReader;import java.io.File;public class DirSpider {private String indexDir;private String sSourceDir;// 控制显示配置参数private boolean verbose;// 控制是否增量索引private boolean incremental;private IndexWriter index;// 从主函数的参数中得到所需配置信息public static void main(String[] args) throws Exception {DirSpider s = new DirSpider(args);s.go();}public DirSpider(String argv[]) throws Exception {verbose = false;incremental = false;for (int i = 0; i < argv.length; i++) {if (argv[i].equals("-s"))sSourceDir = argv[++i];if (argv[i].equals("-d"))indexDir = argv[++i];else if (argv[i].equals("-v"))verbose = true;else if (argv[i].equals("-a"))incremental = true;}if (sSourceDir == null)throw new IllegalArgumentException("Missing required argument: -s [SourceDir dir]");if (indexDir == null)throw new IllegalArgumentException("Missing required argument: -d [index dir]");}public void go() throws Exception {long start = System.currentTimeMillis();// create the index directory -- or append to existingif (verbose) {System.out.println("Creating index in: " + indexDir);if (incremental)System.out.println(" - using incremental mode");}index = new IndexWriter(new File(indexDir), new StandardAnalyzer(),!incremental);File dir = new File(sSourceDir);indexDir(dir);index.optimize();index.close();if (verbose)System.out.println("index complete in :"+ (System.currentTimeMillis() - start) / 1000);}private void indexDir(File dir) {File[] files = dir.listFiles();for (int i = 0; i < files.length; i++) {File f = files[i];if (f.isDirectory()) {indexDir(f); // recurse 递归调用} else if (f.getName().endsWith(".txt")) {indexFile(f);}}}private void indexFile(File item) {if (verbose)System.out.println("Adding FILE: " + item);News news = loadFile(item);if (news != null && news.body != null) {Document doc = new Document();Field f = new Field("url", news.URL, Field.Store.YES,Field.Index.UN_TOKENIZED, Field.TermVector.NO);doc.add(f);f = new Field("title", news.title, Field.Store.YES,Field.Index.TOKENIZED,Field.TermVector.WITH_POSITIONS_OFFSETS);doc.add(f);f = new Field("body", news.body.toString(), Field.Store.YES,Field.Index.TOKENIZED,Field.TermVector.WITH_POSITIONS_OFFSETS);doc.add(f);// 要清晰异常是什么,该怎样去处理。try {index.addDocument(doc);} catch (Exception e) {e.printStackTrace();// System.exit(0)和System.exit(1)分别表示正常退出和异常退出System.exit(-1);}}}private static News loadFile(File sSourceFile) {News news = new News();try { // 注意附加URL的方式news.URL = "http://www.lietu.com/segtest/" + sSourceFile;BufferedReader br = new BufferedReader(new FileReader(sSourceFile));String s;if ((s = br.readLine()) != null) {news.title = s;System.out.println(s);while ((s = br.readLine()) != null) {news.body.append(s);news.body.append('/n');}}br.close();} catch (Exception e) {e.printStackTrace();}return news;}}class News {public String URL;public String title;public StringBuffer body;public News() {this.URL = "";this.title = "";this.body = new StringBuffer();}public String toString() {return "URL :" + URL + " title :" + title + " body :" + body.toString();}}

//suffixArray#include <iostream> #include <cstring> #include <algorithm> using namespace std; #define maxn 200010 int k, n; int s[maxn], height[maxn], h[maxn], rk[maxn], sa[maxn], rdx[maxn]; int next(const int &a) { return a + k < n ? rk[a + k] : 0; } bool cmp(const int &a, const int &b) { return s[a] < s[b]; } void suffixArray() { for(int i = 0; i < n; i++) sa[i] = i; sort(sa, sa + n, cmp); rk[n - 1] = 0; for(int i = 1, j = 0; i < n; i++) { if(s[sa[i]] != s[sa[i - 1]]) j++; rk[sa[i]] = j; } for(k = 1; k < n; k *= 2) { memset(rdx, 0, sizeof(rdx)); for(int i = 0; i < n; i++) rdx[next(i)]++; for(int i = 1; i < n; i++) rdx[i] += rdx[i - 1]; for(int i = 0; i < n; i++) sa[--rdx[next(i)]] = i; memset(rdx, 0, sizeof(rdx)); for(int i = 0; i < n; i++) rdx[rk[i]]++; for(int i = 1; i < n; i++) rdx[i] += rdx[i - 1]; for(int i = n - 1; i >= 0; i--) h[--rdx[rk[sa[i]]]] = sa[i]; memcpy(sa, h, n * sizeof(int)); h[n - 1] = 0; for(int i = 1, j = 0; i < n; i++) { if(rk[sa[i]] != rk[sa[i - 1]] || next(sa[i]) != next(sa[i - 1])) j++; h[sa[i]] = j; } memcpy(rk, h, n * sizeof(int)); } height[0] = 0; for(int i = 0, j = 0; i < n - 1; i++) { while(s[sa[rk[i] - 1] + j] == s[i + j]) j++; height[rk[i]] = j; if(j > 0) j--; } }int r[maxn][20];void initRMQ() {for(int i = 0; i < n; i++) r[i][0] = height[i];for(int j = 1; (1 << j) <= n; j++) for(int i = 0; i + (1 << j) <= n; i++) r[i][j] = min(r[i][j - 1], r[i + (1 << (j - 1))][j - 1]);}int RMQ(int a, int b) {int L = (int)(log(b - a + 1.0) / log(2.0));return min(r[a][L], r[b - (1 << L) + 1][L]);}int lcp(int a, int b) {a = rk[a], b = rk[b];if(a > b) swap(a, b);return RMQ(a + 1, b);}