Lucene与Tag图

Tag图是采用了Tag作为文章管理工具的网站经常需要呈现的一种视图。利用Lucene的优异性能，可以出色的完成这一功能。

生成一个Tag图，首先需要知道用于一共使用了哪些Tag，其次需要知道每个Tag被使用的次数。

对于这两个功能，都可以使用Lucene.Index.IndexReader.Terms方法。这个方法返回索引目录下所有Term，以及他们在全部文档中被使用的次数。这就为我们生成Tag提供了必要的基础。但是Terms方法返回的TermEnum的排序算法是按照FieldName,text的方式排序的，而不是按照docfreq排序的，所以需要还实现一个排序算法。

首先是索引的结构。我设计了如下的索引结构：

docurl:文档的url

contents:文档的内容，以便全文索引

doctags:文档相关的所有tags.tag以空格或逗号作为分割，可以使用单独的Analyzer进行解析。可以参考Analyzer以及PerFieldAnalyzerWrapper两个类。
排序算法，使用一个链表作为保存Tag的形式。它的两个方法GetList(int top)和Top(int freq)可以帮助我们设定Tag图中需要包含的Tag。TermFreq是每个Tag的数据内容。TermFreq.term是Tag的内容。TermFreq.freq是被使用的次数，这样就可以设定Tag的显示样式了。链表通过一个SortedList作为帮助信息，以便提高排序的效率。经过测试，这个排序算法对200M的TermFreq只需要11秒的时间。

  1 internal class TermFreq
  2         {
  3             public string term;
  4             public int freq = 0;
  5         }
  6         internal class TermFreqCompare : System.Collections.IComparer
  7         {
  8             #region IComparer 成员
  9 
 10             public int Compare(object x, object y)
 11             {
 12                 TermFreq f1 = x as TermFreq;
 13                 TermFreq f2 = y as TermFreq;
 14                 int compareResult = f1.freq.CompareTo(f2.freq);
 15                 //if(compareResult==0) return f2.term.CompareTo(f1.term);
 16                 return compareResult;
 17             }
 18 
 19             #endregion
 20 
 21         }
 22         internal class TermFreqSortedList
 23         {
 24             private Element root;
 25             private System.Collections.IComparer comparer;
 26             private System.Collections.SortedList list;
 27             internal class Element
 28             {
 29                 public Element prev;
 30                 public Element next;
 31                 public TermFreq current;
 32             }
 33             public TermFreqSortedList(System.Collections.IComparer comparer)
 34             {
 35                 root = new Element();
 36                 root.current = new TermFreq();
 37                 this.comparer = comparer;
 38                 list = new System.Collections.SortedList();
 39             }
 40             private Element GetStartElement(int freq)
 41             {
 42                 Element ele = null;
 43                 if(list.ContainsKey(freq))
 44                 {
 45                     ele = list[freq] as Element;
 46                 }
 47                 else
 48                 {
 49                     list.Add(freq,null);
 50                     int index = list.IndexOfKey(freq)-1;
 51                     if(index<0) ele = list[0] as Element;
 52                     else ele = list[index] as Element;
 53                 }
 54                 return ele;
 55             }
 56             public void Add(TermFreq o)
 57             {
 58                 Element ele = GetStartElement(o.freq);
 59                 if(ele==null) ele = root;
 60                 Element oEle = new Element();
 61                 oEle.current = o;
 62                 list[oEle.current.freq] = oEle;
 63                 while(ele !=null)
 64                 {
 65                     int compareResult = comparer.Compare(ele.current,oEle.current);
 66                     if(compareResult>0)
 67                     {
 68                         if(ele.next==null)
 69                         {
 70                             ele.next = oEle;
 71                             oEle.prev = ele;
 72                             break;
 73                         }
 74                         else if(comparer.Compare(ele.next.current,oEle.current)<0)
 75                         {
 76                             ele.next.prev = oEle;
 77                             oEle.next = ele.next;
 78                             ele.next = oEle;
 79                             oEle.prev = ele;
 80                             break;
 81                         }
 82                         else
 83                         {
 84                             ele = ele.next;
 85                             continue;
 86                         }
 87                     }
 88                     else if(compareResult<0)
 89                     {
 90                         if(ele.prev==null)
 91                         {
 92                             ele.prev = oEle;
 93                             oEle.next = ele;
 94                             root = oEle;
 95                             break;
 96                         }
 97                         else if(comparer.Compare(ele.prev.current,oEle.current)>0)
 98                         {
 99                             ele.prev.next = oEle;
100                             oEle.prev = ele.prev;
101 
102                             ele.prev = oEle;
103                             oEle.next = ele;
104                             break;
105                         }
106                         else
107                         {
108                             ele = ele.prev;
109                             continue;
110                         }
111                     }
112                     if(ele.prev!=null)
113                     {
114                         ele.prev.next = oEle;
115                         oEle.prev = ele.prev;
116                     }
117                     else
118                     {
119                         root = oEle;
120                     }
121                     oEle.next = ele;
122                     ele.prev = oEle;
123                     break;
124                 }
125             }
126             public System.Collections.ArrayList GetList(int top)
127             {
128                 System.Collections.ArrayList list = new System.Collections.ArrayList();
129                 Element ele = root;
130                 int i=0;
131                 while((i++)<top)
132                 {
133                     list.Add(ele.current);
134                     if(ele.next == null)
135                     {
136                         return list;
137                     }
138                     ele = ele.next;
139                 }
140                 return list;
141             }
142             public System.Collections.ArrayList Top(int freq)
143             {
144                 System.Collections.ArrayList list = new System.Collections.ArrayList();
145                 Element ele = root;
146                 while(ele.current.freq >= freq)
147                 {
148                     list.Add(ele.current);;
149                     if(ele.next==null)
150                         return list;
151                     ele = ele.next;
152                 }
153                 return list;
154             }
155         }

文档生成的代码：

 

1 Document doc = new Document();        
2 doc.Add(Field.Keyword("docurl", docurl));
3 doc.Add(Field.Text("contents",contents));
4 //storeTermVector==true.这样我们以后就可以通过TermFreqVector来访问tag在每个文档中被标注的次数了，以便生成单个文档的Tag图
5 doc.Add(Field.Text("doctags", reader,true));

测试代码:

 1 Lucene.Net.Index.TermEnum enu = reader.Terms(new Term("contents","_"));
 2                 TermFreqSortedList list = new TermFreqSortedList(new TermFreqCompare());
 3                 
 4                 while(enu.Next())
 5                 {
 6                     Lucene.Net.Index.Term t = enu.Term();
 7                     
 8                     TermFreq f = new TermFreq();
 9                     f.freq = enu.DocFreq();
10                     f.term = t.Text();
11                     list.Add(f);
12                 }
13                 for(System.Collections.IEnumerator ienu = list.GetList(5).GetEnumerator();ienu.MoveNext();)
14                 {
15                     TermFreq ff = ienu.Current as TermFreq;
16                     
17                     Console.WriteLine(string.Format("Term:{0}./t/t/tDocFreq:{1}",
18                         ff.term,
19                         ff.freq));
20                 }
21                 for(System.Collections.IEnumerator ienu = list.Top(3).GetEnumerator();ienu.MoveNext();)
22                 {
23                     TermFreq ff = ienu.Current as TermFreq;
24                     
25                     Console.WriteLine(string.Format("Term:{0}./t/t/tDocFreq:{1}",
26                         ff.term,
27                         ff.freq));
28                 }