文档的倒排 invertDocument方法的源代码

代码中的Enumeration如果有不懂得，参见我的上一篇博客。

package indexwriter;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Enumeration;
import java.lang.Object.*;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.TermVectorOffsetInfo;


 //invertDocument方法的源代码 
public class InvertDocument {

private final void invertDocument(Document doc) throws IOException{
FieldInfo
//获取对Document中所有field的遍历
Enumeration fields = doc.fields();
//判断是否还有元素
while(fields.hasMoreElements()){
Field field = (Field) fields.nextElement();

//取出field的名称
String fileName = field.name();

//取出field的编号
int fileNumber = fieldInfos.fileNumber(fieldName);

int length = fieldLengths[fieldNumber];
int position = fieldPositions[fieldNumber];
if(length > 0)
position += analyzer.getPositionIncrementGap(fileName);
int offset = fieldOffsets[fieldNumber];

//如果当前field是需要被索引的
if(field.isIndexed()){
//首先处理那些不需要被分词的field
if(!field.isTokenized()){

//取出当前的field的数值
String stringValue = field.stringValue();

//是否按词条向量存储
if(field.isStoreOffsetWithTermVector())
//把整个field的数据当成一个词条存储进postingTable中
addPosition(fileName, stringValue, position++, new TermVectorOffsetInfo(offset, offset + stringValue.length()));
else
//把整个field的数据当成一个词条进行存储进postingTable中
addPostion(fieldName, stringValue, postion++, null);
offset += stringValue.length();
length++;
}

//如field需要进行分词
else{
Reader reader;
//如果是用Reader来当做数据源的
if(field.readerValue() != null)
reader = field.readerValue();
//如果是直接用字符串来当做数据的
else if(field.stringValue() !=null)
reader= new StringReader(field.stringValue());
else
throw new IllegalArgumentException("field must have either String or Reader value()");

//调用底层分析器的接口来对field里的数据进行分词和过滤
TokenStream stream = analyzer.tokenStream(fieldName, reader);

try{
Token lastToken = null;
for(Token t = stream.next(); t != null; t = stream.next()){
//每切出一个词，就将position加上当前这个切出来的词的长度
position += (t.getPositionIncrement() - 1);

//如果与词条向量有关的就存储偏移量信息
if(field.isStoreOffsetWithTermVector())

//将切出来的词存入postingTable中
addPosition(fieldName, t.termText(), position++, new TermVectorOffsetInfo(offset + t.startOffset(), offset + t.endOffset()));
else
//将切出的词条存入postingTable中
addPosition(fieldName, t.termText(), position++, null);
lastToken = t;

//如果当前的词条数目已经达到了最大域长度
if(++length > maxFieldLength){
if(infoStream != null)
infoStream.println("maxFieldLength " + maxFieldLength + " reached, ignoring following tokens");
break;

}

}

if(lastToken != null)
offset += lastToken.endOffset() + 1;

}finally{
stream.close();
}


}

//记录先关属性信息
fieldLengths[fieldNumber] = length;
fieldPosition[fieldNumber] = position;

fieldOffsets[fieldNumber] = offset;

}



}

}


}