带索引的HIVE

    hive的查询跟普通的hadoop mapreduce没有什么大的区别，都是对原始数据的暴力扫描，如果能够像数据库那样，使用索引，那么数据扫描的速度将会大幅度提升
上次在mapreduce上使用了索引，具体参见下面这个链接
http://user.qzone.qq.com/165162897/blog/1351432946
这次在这个基础上拓展到hive里（实际上也是一个特殊的inputformat），使用示例参见如下


一、创建索引（这个没啥好说的，直接看后面的源码吧）
hadoop jar ./higo-manager-1.3.1-SNAPSHOT.jar com.alipay.higo.hadoop.sequenceIndex.SequenceIndexExample create /group/tbdev/lingning/yannian.mu/input/1.txt /group/tbdev/lingning/yannian.mu/output 20

二、创建hive表（除了inputformat外，没任何特别之处）
CREATE EXTERNAL TABLE yannian_hive_index_test(col1 String,col2 String,col3 String,col4 String,col5 String)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001'
STORED AS INPUTFORMAT 'com.alipay.higo.hadoop.sequenceIndex.SequenceIndexInputFormatForHive'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION '/group/tbdev/lingning/yannian.mu/output'

四、添加必要的jar
add jar ./higo-index-1.3.1-SNAPSHOT.jar;   //inputformat程序
add jar ./lucene-core-3.5-SNAPSHOT.jar ;//依赖的lucene 

五、查询前的基于lucene索引的过滤
//设置hive表字段，一定要与创建表时候的字段顺序和个数一致
set hive.fields.sequence=col1,col2,col3,col4,col5;
//设置本地查询用到的字段，只有用到的字段才会被扫描
set lucene.fields=col1,col3,col2;
//lucene的查询条件-这里表示仅仅扫描col1字段前缀为1的数据行
set lucene.query=col1:1*;
 
六、经过lucene过滤后的结果，使用HIVE继续进行分析
select col1,col3 from yannian_hive_index_test limit 1000;

不算太麻烦吧，这回贴下完整的实现代码。

package com.alipay.higo.hadoop.sequenceIndex;import java.io.DataInput;import java.io.DataOutput;import java.io.FileNotFoundException;import java.io.IOException;import java.rmi.server.UID;import java.security.MessageDigest;import java.util.Arrays;import java.util.HashMap;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.ChecksumException;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.DataOutputBuffer;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.VersionMismatchException;import org.apache.hadoop.io.SequenceFile.Metadata;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.util.Progressable;import org.apache.lucene.store.BufferedIndexInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.store.Lock;/** * 基于lucene索引的顺序索引  *  * @author yannian.mu * */public class SequenceIndex { private static final Log LOG = LogFactory.getLog(SequenceIndex.class); private static final byte VERSION_WITH_METADATA = (byte) 6; private static final int SYNC_ESCAPE = -1; // "length" of sync entries private static final int SYNC_HASH_SIZE = 16; // number of bytes in hash private static final int SYNC_SIZE = 4 + SYNC_HASH_SIZE; // escape + hash public static final int SYNC_INTERVAL = 100 * SYNC_SIZE; private static byte[] VERSION = new byte[] { (byte) 'S', (byte) 'E',   (byte) 'I', VERSION_WITH_METADATA };  public static Writer create(FileSystem fs, Configuration conf, Path name,   int bufferSize, short replication, long blockSize,   Progressable progress, Metadata metadata) throws IOException {  return new Writer(fs, conf, name, bufferSize, replication, blockSize, progress, metadata); }  public static Reader open(FileSystem fs, Path file, int bufferSize, long start,   long length, Configuration conf, boolean tempReader) throws IOException {  return new Reader(fs, file, bufferSize, start, length, conf, tempReader); } public static class Writer implements java.io.Closeable {  Configuration conf;  FSDataOutputStream out;  boolean ownOutputStream = true;  DataOutputBuffer buffer = new DataOutputBuffer();  Metadata metadata = null;  long lastSyncPos; // position of last sync  byte[] sync; // 16 random bytes  {   try {    MessageDigest digester = MessageDigest.getInstance("MD5");    long time = System.currentTimeMillis();    digester.update((new UID() + "@" + time).getBytes());    sync = digester.digest();   } catch (Exception e) {    throw new RuntimeException(e);   }  }  public Writer(FileSystem fs, Configuration conf, Path name,    int bufferSize, short replication, long blockSize,    Progressable progress, Metadata metadata) throws IOException {   this.conf = conf;   this.out = fs.create(name, true, bufferSize, replication,     blockSize, progress);   this.metadata = metadata;   out.write(VERSION);   this.metadata.write(out);   out.write(sync); // write the sync bytes   out.flush();  }  public void sync() throws IOException {   if (sync != null && lastSyncPos != out.getPos()) {    out.writeInt(SYNC_ESCAPE); // mark the start of the sync    out.write(sync); // write sync    lastSyncPos = out.getPos(); // update lastSyncPos   }  }  public Configuration getConf() {   return conf;  }  public synchronized void close() throws IOException {   if (out != null) {    if (ownOutputStream) {     out.close();    } else {     out.flush();    }    out = null;   }  }  synchronized void checkAndWriteSync() throws IOException {   if (sync != null && out.getPos() >= lastSyncPos + SYNC_INTERVAL) { // time    sync();   }  }  public synchronized void append(Text key, Directory dir)    throws IOException {   checkAndWriteSync();   String[] names=dir.listAll();   out.writeInt(key.getLength());   out.write(key.getBytes(), 0, key.getLength());   out.writeInt(names.length);   for (String name : dir.listAll()) {    Text nameText=new Text(name);    out.writeInt(nameText.getLength());    out.write(nameText.getBytes(), 0,nameText.getLength());    long filelen=dir.fileLength(name);    out.writeLong(filelen);    this.writeTo(filelen, dir.openInput(name.toString()), out);   }   }      private void writeTo(long end,IndexInput input,FSDataOutputStream out) throws IOException {       long pos = 0;       int bufflen=1024;       while (pos < end) {         int length = bufflen;         long nextPos = pos + length;         if (nextPos > end) {                        // at the last buffer           length = (int)(end - pos);         }        byte[] buff=new byte[length];        input.readBytes(buff, 0, length);                  out.write(buff,0,length);         pos = nextPos;       }     }  public synchronized long getLength() throws IOException {   return out.getPos();  } } public static class Reader implements java.io.Closeable {  private Path file;  private FSDataInputStream in;  private FSDataInputStream shardIn;  private byte version;  private Metadata metadata = null;  private byte[] sync = new byte[SYNC_HASH_SIZE];  private byte[] syncCheck = new byte[SYNC_HASH_SIZE];  private boolean syncSeen;  private long end;  private Configuration conf;    private Reader(FileSystem fs, Path file, int bufferSize, long start,    long length, Configuration conf, boolean tempReader)    throws IOException {   this.file = file;   this.in = fs.open(file, bufferSize);   this.shardIn=fs.open(file, bufferSize);   this.conf = conf;   seek(start);   this.end = in.getPos() + length;   init(tempReader);  }  private void init(boolean tempReader) throws IOException {   byte[] versionBlock = new byte[VERSION.length];   in.readFully(versionBlock);   if ((versionBlock[0] != VERSION[0])     || (versionBlock[1] != VERSION[1])     || (versionBlock[2] != VERSION[2]))    throw new IOException(file + " not a SequenceIndex");   version = versionBlock[3];   if (version > VERSION[3])    throw new VersionMismatchException(VERSION[3], version);   this.metadata = new Metadata();   if (version >= VERSION_WITH_METADATA) { // if version >= 6    this.metadata.readFields(in);   }   if (version > 1) { // if version > 1    in.readFully(sync); // read sync bytes   }  }  public synchronized void close() throws IOException {   in.close();   this.shardIn.close();  }  private synchronized int readKeyLength() throws IOException {   if (in.getPos() >= end) {    return -1;   }   int length = in.readInt();   if (version > 1 && sync != null && length == SYNC_ESCAPE) { // process    in.readFully(syncCheck); // read syncCheck    if (!Arrays.equals(sync, syncCheck)) // check it     throw new IOException("File is corrupt!");    syncSeen = true;    if (in.getPos() >= end) {     return -1;    }    length = in.readInt(); // re-read length   } else {    syncSeen = false;   }   return length;  }  public synchronized int next(Text key, SequenceIndexDirectory dir)    throws IOException {   int length = readKeyLength();   if (length == -1) {    return -1;   }      dir.setShareStream(this.shardIn);      byte[] keydata = new byte[length];   in.read(keydata, 0, length);   key.set(keydata);   int filecount = in.readInt();   for (int i = 0; i < filecount; i++) {    int namelen = in.readInt();    byte[] namebyte = new byte[namelen];    in.read(namebyte, 0, namelen);    Text name = new Text(namebyte);    long filelen = in.readLong();    long pos = in.getPos();    in.skip(filelen);    dir.addFile(name.toString(), pos, filelen);   }   return length;  }  public Metadata getMetadata() {   return this.metadata;  }  Configuration getConf() {   return conf;  }  public synchronized void seek(long position) throws IOException {   in.seek(position);  }  public synchronized void sync(long position) throws IOException {   if (position + SYNC_SIZE >= end) {    seek(end);    return;   }   try {    seek(position + 4); // skip escape    in.readFully(syncCheck);    int syncLen = sync.length;    for (int i = 0; in.getPos() < end; i++) {     int j = 0;     for (; j < syncLen; j++) {      if (sync[j] != syncCheck[(i + j) % syncLen])       break;     }     if (j == syncLen) {      in.seek(in.getPos() - SYNC_SIZE); // position before               // sync      return;     }     syncCheck[i % syncLen] = in.readByte();    }   } catch (ChecksumException e) { // checksum failure    handleChecksumException(e);   }  }  private void handleChecksumException(ChecksumException e)    throws IOException {   if (this.conf.getBoolean("io.skip.checksum.errors", false)) {    LOG.warn("Bad checksum at " + getPosition() + ". Skipping entries.");    sync(getPosition()+ this.conf.getInt("io.bytes.per.checksum", 512));   } else {    throw e;   }  }  public boolean syncSeen() {   return syncSeen;  }  public synchronized long getPosition() throws IOException {   return in.getPos();  }  public String toString() {   return file.toString();  } }   public static class HadoopDirectory implements WritableComparable{  Directory dir=null;  public Directory getDir() {   return dir;  }  public void setDir(Directory dir) {   this.dir = dir;  }  @Override  public void write(DataOutput out) throws IOException {   throw new UnsupportedOperationException();  }  @Override  public void readFields(DataInput in) throws IOException {   throw new UnsupportedOperationException();  }  @Override  public int compareTo(Object arg0) {   throw new UnsupportedOperationException();  } } public static class SequenceIndexDirectory extends Directory {  private static int BUFFER_SIZE = 1024;  private static final class FileEntry {   long offset;   long length;   public FileEntry(long offset, long length) {    this.offset = offset;    this.length = length;   }  }  private FSDataInputStream shareStream;  private HashMap<String, FileEntry> entries = new HashMap<String, FileEntry>();  @Override  public synchronized void close() throws IOException {   if (shareStream == null)    throw new IOException("Already closed");   entries.clear();   shareStream = null;  }  public void setShareStream(FSDataInputStream _stream) {   this.shareStream = _stream;  }  public void addFile(String name, long offset, long length) {   entries.put(name, new FileEntry(offset, length));  }  @Override  public synchronized IndexInput openInput(String id) throws IOException {   return openInput(id, BUFFER_SIZE);  }  @Override  public synchronized IndexInput openInput(String id, int readBufferSize)    throws IOException {   if (shareStream == null)    throw new IOException("Stream closed");   FileEntry entry = entries.get(id);   if (entry == null) {    throw new IOException("No sub-file with id " + id      + " found (files: " + entries.keySet() + ")");   }   return new ShareIndexInput(id,shareStream, entry.offset,     entry.length);  }  @Override  public String[] listAll() {   return entries.keySet().toArray(new String[entries.size()]);  }  @Override  public boolean fileExists(String name) {   return entries.containsKey(name);  }  @Override  public long fileModified(String name) throws IOException {   throw new UnsupportedOperationException();  }  @Override  @Deprecated  public void touchFile(String name) throws IOException {   throw new UnsupportedOperationException();  }  @Override  public void deleteFile(String name) {   throw new UnsupportedOperationException();  }  public void renameFile(String from, String to) {   throw new UnsupportedOperationException();  }  @Override  public long fileLength(String name) throws IOException {   FileEntry e = entries.get(name);   if (e == null)    throw new FileNotFoundException(name);   return e.length;  }  @Override  public IndexOutput createOutput(String name) {   throw new UnsupportedOperationException();  }  @Override  public Lock makeLock(String name) {   throw new UnsupportedOperationException();  }  public static class ShareIndexInput extends BufferedIndexInput {   public class Descriptor{        FSDataInputStream in=null;    public FSDataInputStream getIn() {     return in;    }    public void setIn(FSDataInputStream in) {     this.in = in;    }        public void close()    {         }   }   private final Descriptor descriptor;   private final long length;   @Override   public String toString() {    return "ShareIndexInput [length=" + length + ", fileOffset="      + fileOffset + ", filename=" + filename + "]";   }   private boolean isOpen;   private boolean isClone;   private long fileOffset;   private String filename;   public ShareIndexInput(String _filename,FSDataInputStream shareStream,     long _fileOffset, long _length) throws IOException {    super("sequenceIndex input");    this.filename=_filename;    this.descriptor = new Descriptor();    this.descriptor.setIn(shareStream);        this.fileOffset = _fileOffset;    this.length = _length;    this.isOpen = true;    this.isClone = false;   }   protected void readInternal(byte[] b, int offset, int len)     throws IOException {    synchronized (descriptor.in) {     long position = getFilePointer();     if ((position+this.fileOffset) != descriptor.in.getPos()) {      descriptor.in.seek(position+this.fileOffset);     }     int total = 0;     do {      int i = descriptor.in.read(b, offset + total, len        - total);      if (i == -1) {       throw new IOException("Read past EOF");      }      total += i;     } while (total < len);    }   }   public void close() throws IOException {    if (!isClone) {     if (isOpen) {      descriptor.close();      isOpen = false;     } else {      throw new IOException("Index file already closed");     }    }   }   public long length() {    return length;   }   protected void finalize() throws IOException {    if (!isClone && isOpen) {     close();    }   }   public Object clone() {    ShareIndexInput clone = (ShareIndexInput) super.clone();    clone.isClone = true;    return clone;   }   @Override   protected void seekInternal(long pos) throws IOException {       }     }   }}###################################################################################package com.alipay.higo.hadoop.sequenceIndex;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.BytesWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.JobClient;import org.apache.hadoop.mapred.JobConf;import org.apache.hadoop.mapred.OutputCollector;import org.apache.hadoop.mapred.Reporter;import org.apache.hadoop.mapred.RunningJob;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.Fieldable;import org.apache.lucene.document.Field.Index;import org.apache.lucene.document.Field.Store;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;import org.apache.lucene.index.IndexWriter.MaxFieldLength;import org.apache.lucene.queryParser.ParseException;import org.apache.lucene.queryParser.QueryParser;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.ScoreDoc;import org.apache.lucene.search.TopDocs;import org.apache.lucene.store.Directory;import org.apache.lucene.store.RAMDirectory;import org.apache.lucene.util.Version;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndex.HadoopDirectory;public class SequenceIndexExample { public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {  String type=args[0];  String input=args[1];  String output=args[2];  Integer numreduce=Integer.parseInt(args[3]);  if(type.equals("create"))  {   create(input, output,numreduce);  }if(type.equals("searchold"))  {   searchOld(input, output,numreduce);  }else{   search(input, output,numreduce);  } } private static void search(String input,String output,int numreduce) throws IOException, InterruptedException, ClassNotFoundException {  Job job = new Job(new Configuration());  job.setInputFormatClass(SequenceIndexInputFormat.class);  SequenceIndexInputFormat.addInputPath(job, new Path(input));  job.setMapperClass(IndexMap.class);  job.setJarByClass(SequenceIndexExample.class);  job.setOutputKeyClass(Text.class);  job.setOutputValueClass(Text.class);  TextOutputFormat.setOutputPath(job, new Path(output));  job.setNumReduceTasks(numreduce);  job.waitForCompletion(true); }  private static void searchOld(String input,String output,int numreduce) throws IOException, InterruptedException, ClassNotFoundException {  Configuration conf=new Configuration();  conf.set("hive.fields.sequence","index,col1,col2,col3,col4,col5");  conf.set("lucene.fields","index,col3");  conf.set("lucene.query","index:500");  JobConf jobconf=new JobConf(conf, SequenceIndexInputFormatForHive.class);  jobconf.setJobName("oldsearch");  jobconf.setNumReduceTasks(numreduce);  jobconf.setInputFormat(SequenceIndexInputFormatForHive.class);  jobconf.setMapperClass(OldMapper.class);    jobconf.setOutputKeyClass(Text.class);  jobconf.setOutputValueClass(Text.class);  SequenceIndexInputFormatForHive.addInputPath(jobconf, new Path(input));  org.apache.hadoop.mapred.FileOutputFormat.setOutputPath(jobconf,new Path(output));        RunningJob rj = JobClient.runJob(jobconf); }   public static class OldMapper implements org.apache.hadoop.mapred.Mapper<LongWritable, BytesWritable, Text, Text> {  @Override  public void configure(JobConf job) {     }  @Override  public void close() throws IOException {     }  @Override  public void map(LongWritable key, BytesWritable value,    OutputCollector<Text, Text> output, Reporter reporter)    throws IOException {   output.collect(new Text(String.valueOf(key.get())), new Text(value.get()));     }           }  private static void create(String input,String output,int numreduce) throws IOException, InterruptedException, ClassNotFoundException {  Job job = new Job(new Configuration());  FileInputFormat.addInputPath(job, new Path(input));  job.setJarByClass(SequenceIndexExample.class);  job.setMapOutputKeyClass(LongWritable.class);  job.setMapOutputValueClass(Text.class);    job.setReducerClass(IndexReducer.class);  job.setOutputKeyClass(Text.class);  job.setOutputValueClass(HadoopDirectory.class);  job.setOutputFormatClass(SequenceIndexOutputFormat.class);  SequenceIndexOutputFormat.setOutputPath(job, new Path(output));  job.setNumReduceTasks(numreduce);  job.waitForCompletion(true); }  public static class IndexMap extends   Mapper<Text, HadoopDirectory, Text, Text> {  protected void map(Text key, HadoopDirectory value, Context context)    throws IOException, InterruptedException {   Directory dir = value.getDir();   IndexReader reader = IndexReader.open(dir);   StandardAnalyzer an = new StandardAnalyzer(Version.LUCENE_35);   QueryParser q = new QueryParser(Version.LUCENE_35, "index", an);   IndexSearcher searcher = new IndexSearcher(reader);   TopDocs docs;   try {    docs = searcher.search(q.parse("index:500"), 20);   } catch (ParseException e) {    throw new RuntimeException(e);   }   ScoreDoc[] list = docs.scoreDocs;   if (list != null && list.length > 0) {    StringBuffer buff = new StringBuffer();    for (ScoreDoc doc : list) {     Document document = searcher.doc(doc.doc);     for (Fieldable f : document.getFields()) {      buff.append(f.name() + "="        + document.getFieldable(f.name()).stringValue()        + ",");     }     context.write(key, new Text(buff.toString()));    }   }  } } public static class IndexReducer extends    Reducer<LongWritable, Text, Text, HadoopDirectory> {  boolean setup=false;    protected void reduce(LongWritable key, Iterable<Text> values,        Context context) throws java.io.IOException, InterruptedException {   if(setup)   {    return;   }   setup=true;     for(int k=0;k<10000;k++)     {      HadoopDirectory hdir=new HadoopDirectory();      hdir.setDir(new RAMDirectory());            IndexWriter writer = new IndexWriter(hdir.getDir(), null,             new KeepOnlyLastCommitDeletionPolicy(),             MaxFieldLength.UNLIMITED);     writer.setUseCompoundFile(false);     writer.setMergeFactor(2);    System.out.println(k);        for(int i=0;i<1000;i++)    {     Document doc=new Document();     doc.add(new Field("index", String.valueOf(i), Store.YES, Index.NOT_ANALYZED_NO_NORMS));     for(int j=0;j<10;j++)     {      doc.add(new Field("col"+j, String.valueOf(i)+","+j+","+k, Store.YES, Index.NOT_ANALYZED_NO_NORMS));     }     writer.addDocument(doc);    }        writer.optimize();    writer.close();    context.write(new Text(String.valueOf(k)), hdir);     }         }   }}#####################################################################package com.alipay.higo.hadoop.sequenceIndex;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.InputSplit;import org.apache.hadoop.mapreduce.RecordReader;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.FileSplit;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndex.HadoopDirectory;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndex.SequenceIndexDirectory;public class SequenceIndexInputFormat extends FileInputFormat<Text,HadoopDirectory>{ @Override   public RecordReader<Text,HadoopDirectory> createRecordReader(InputSplit split,                                                TaskAttemptContext context                                                ) throws IOException {  try {   return  new SequenceIndexRecordReader(split,context);  } catch (InterruptedException e) {   throw new IOException(e);  }   }   @Override   protected long getFormatMinSplitSize() {     return SequenceIndex.SYNC_INTERVAL;   }      public static class SequenceIndexRecordReader extends RecordReader<Text,HadoopDirectory>{    private SequenceIndex.Reader in;    private long start;    private long end;    private boolean more = true;    private Text key = null;    private HadoopDirectory value = null;    protected Configuration conf;    public void initialize(InputSplit split,                   TaskAttemptContext context                  ) throws IOException, InterruptedException {    }    public SequenceIndexRecordReader(InputSplit split,                            TaskAttemptContext context                           ) throws IOException, InterruptedException {     FileSplit fileSplit = (FileSplit) split;      this.init(context.getConfiguration(), fileSplit.getPath(), fileSplit.getStart(), fileSplit.getLength());    }          public SequenceIndexRecordReader(Configuration _conf,Path _path,long _start,long _len) throws IOException,    InterruptedException {   this.init(_conf, _path, _start, _len);  }    private void init(Configuration _conf, Path path,    long _start, long len) throws IOException, InterruptedException {   conf = _conf;   FileSystem fs = path.getFileSystem(conf);   this.in = SequenceIndex.open(fs, path, conf.getInt(     "io.file.buffer.size", 4096), 0, fs.getFileStatus(path)     .getLen(), conf, false);// new SequenceFile.Reader(fs, path,   this.end = _start + len;   if (_start > in.getPosition()) {    in.sync(_start); // sync to start   }   this.start = in.getPosition();   more = _start < end;  }    @Override    public boolean nextKeyValue() throws IOException, InterruptedException {      if (!more) {        return false;      }      long pos = in.getPosition();            this.key=new Text();      this.value=new HadoopDirectory();      SequenceIndexDirectory dir=new SequenceIndexDirectory();      this.value.setDir(dir);      if(this.in.next(this.key, dir)<0||(pos >= end && in.syncSeen()))      {       more = false;       key = null;       value = null;      }      return more;    }    @Override    public Text getCurrentKey() {      return key;    }        @Override    public HadoopDirectory getCurrentValue() {      return value;    }        public long getpos() throws IOException    {     return in.getPosition();    }        public float getProgress() throws IOException {      if (end == start) {        return 0.0f;      } else {        return Math.min(1.0f, (in.getPosition() - start) / (float)(end - start));      }    }        public synchronized void close() throws IOException { in.close(); }      }}#############################################################package com.alipay.higo.hadoop.sequenceIndex;import java.io.IOException;import java.util.ArrayList;import java.util.Arrays;import java.util.HashMap;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.io.BytesWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.mapred.FileSplit;import org.apache.hadoop.mapred.InputSplit;import org.apache.hadoop.mapred.JobConf;import org.apache.hadoop.mapred.RecordReader;import org.apache.hadoop.mapred.Reporter;import org.apache.hadoop.mapred.SequenceFileInputFormat;import org.apache.lucene.analysis.standard.StandardAnalyzer;import org.apache.lucene.document.Document;import org.apache.lucene.document.Fieldable;import org.apache.lucene.document.MapFieldSelector;import org.apache.lucene.index.IndexReader;import org.apache.lucene.queryParser.ParseException;import org.apache.lucene.queryParser.QueryParser;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.ScoreDoc;import org.apache.lucene.search.TopDocs;import org.apache.lucene.util.Version;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndex.HadoopDirectory;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndexInputFormat.SequenceIndexRecordReader;public class SequenceIndexInputFormatForHive extends SequenceFileInputFormat<LongWritable, BytesWritable> { public RecordReader<LongWritable, BytesWritable> getRecordReader(   InputSplit split, JobConf job, Reporter reporter)   throws IOException {  FileSplit part = (FileSplit) split;  return new HiveTarRecordReader(job, part); } public static class HiveTarRecordReader implements   RecordReader<LongWritable, BytesWritable> {  private SequenceIndexRecordReader seqReader = null;  IndexReader reader=null;  private String hive_fields = "";  private ArrayList<String> rowFields = new ArrayList<String>();  private ArrayList<String> lucene_fields = new ArrayList<String>();  private String lucene_query = "";  private HadoopDirectory dir;  private IndexSearcher searcher;  private ScoreDoc[] list;  private int lineIndex = -1;  FileSplit split;  public HiveTarRecordReader(Configuration conf, FileSplit _split)    throws IOException {   this.hive_fields = conf.get("hive.fields.sequence","");   this.split=_split;   for(String f:this.hive_fields.split(","))   {    this.rowFields.add(f);   }      for(String f:conf.get("lucene.fields","").split(","))   {    this.lucene_fields.add(f);   }      this.lucene_query = conf.get("lucene.query");   try {    seqReader = new SequenceIndexRecordReader(conf,_split.getPath(),_split.getStart(),_split.getLength());   } catch (InterruptedException e) {    throw new IOException(e);   }  }  public synchronized boolean next(LongWritable pos, BytesWritable k)    throws IOException {   while (lineIndex == -1 || list == null || lineIndex >= list.length) {    try {     if (!seqReader.nextKeyValue()) {      return false;     }    } catch (InterruptedException e1) {     throw new IOException(e1);    }        if(this.searcher!=null)    {     this.searcher.close();    }    if(this.reader!=null)    {     this.reader.close();    }    if(this.dir!=null)    {     this.dir.getDir().close();    }        this.dir = seqReader.getCurrentValue();    try{     this.reader = IndexReader.open(dir.getDir());    }catch(IOException e)    {     throw new IOException(this.split.toString()+"@@@"+dir.getDir().toString()+"@@@"+dir.getDir().getClass().getName(), e);    }    StandardAnalyzer an = new StandardAnalyzer(Version.LUCENE_35);    QueryParser q = new QueryParser(Version.LUCENE_35, "index", an);    this.searcher = new IndexSearcher(reader);    TopDocs docs;    try {     docs = this.searcher.search(q.parse(this.lucene_query), 10000000);    } catch (ParseException e) {     throw new RuntimeException(e);    }    this.list = docs.scoreDocs;    this.lineIndex=0;   }         ScoreDoc doc=this.list[this.lineIndex];   Document document = this.searcher.doc(doc.doc,new MapFieldSelector(this.lucene_fields));   HashMap<String,String> val=new HashMap<String,String>();   for (Fieldable f : document.getFields()) {    String fname=f.name();    val.put(fname, document.getFieldable(fname).stringValue());   }      StringBuffer buff = new StringBuffer();   String joinchar="";   for(String f:this.rowFields)   {    buff.append(joinchar);        if(val.containsKey(f))    {     buff.append(val.get(f));    }else{     buff.append("-");    }        joinchar="\001";   }         pos.set(this.seqReader.getpos());      String line=buff.toString();            byte[] textBytes = line.getBytes();            int length = line.length();                   k.set(textBytes, 0, textBytes.length);      lineIndex++;   return true;  }        public void close() throws IOException {            seqReader.close();        }        public LongWritable createKey() {            return new LongWritable();        }        public BytesWritable createValue() {            return new BytesWritable();        }        public long getPos() throws IOException {            return seqReader.getpos();        }        public float getProgress() throws IOException {            return seqReader.getProgress();        }    }}###################################################################################package com.alipay.higo.hadoop.sequenceIndex;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.SequenceFile.Metadata;import org.apache.hadoop.mapreduce.RecordWriter;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import com.alipay.higo.hadoop.sequenceIndex.SequenceIndex.HadoopDirectory;public class SequenceIndexOutputFormat extends FileOutputFormat<Text,HadoopDirectory>{   public RecordWriter<Text,HadoopDirectory>           getRecordWriter(TaskAttemptContext context                          ) throws IOException, InterruptedException {     Configuration conf = context.getConfiguration();     Path file = getDefaultWorkFile(context, "");     FileSystem fs = file.getFileSystem(conf);     final SequenceIndex.Writer out = SequenceIndex.create(fs, conf, file, conf.getInt("io.file.buffer.size", 4096),  fs.getDefaultReplication(), fs.getDefaultBlockSize(), null, new Metadata());     return new RecordWriter<Text,HadoopDirectory>() {         public void write(Text key, HadoopDirectory value)           throws IOException {           out.append(key, value.getDir());         }         public void close(TaskAttemptContext context) throws IOException {            out.close();         }       };   }}