Hadoop RecordReader

自定义RecordReader
步骤：
1）继承抽象类RecordReader，实现RecordReader的一个实例
2）实现自定义InputFormat类，重写InputFormat中的createRecordReader（）方法，返回值是自定义的RecordReader实例
3）配置job.setInputFormatClass（）设置自定义的InputFormat实例

RecordReader例子
应用场景：
数据：
10
20
30
40
50
60
70
要求：分别计算奇数行与偶数行数据之和
奇数行总和：10+30+50+70=160
偶数行总和：20+40+60=120
MyRecordReader.java

package com.recordreader;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.InputSplit;import org.apache.hadoop.mapreduce.RecordReader;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileSplit;import org.apache.hadoop.util.LineReader;public class MyRecordReader extends RecordReader<LongWritable, Text> {    private long start ;    private long end ;    private long pos ;//表示行号    private FSDataInputStream fin = null ;    private LongWritable key = null ;    private Text value = null ;    private LineReader reader = null ;    @Override    public void close() throws IOException {        fin.close() ;    }    @Override    public LongWritable getCurrentKey() throws IOException,            InterruptedException {        return key;    }    @Override    public Text getCurrentValue() throws IOException, InterruptedException {        return value;    }    @Override    public float getProgress() throws IOException, InterruptedException {        return 0;    }    @Override    public void initialize(InputSplit inputSplit, TaskAttemptContext context)            throws IOException, InterruptedException {        FileSplit fileSplit = (FileSplit)inputSplit ;        start = fileSplit.getStart() ;        end = start + fileSplit.getLength() ;        Configuration conf = context.getConfiguration() ;        Path path = fileSplit.getPath() ;        FileSystem fs = path.getFileSystem(conf) ;        fin = fs.open(path) ;        fin.seek(start) ;        reader = new LineReader(fin) ;        pos = 1 ;    }    @Override    public boolean nextKeyValue() throws IOException, InterruptedException {        if(key == null){            key = new LongWritable() ;        }        key.set(pos) ;        if(value == null){            value = new Text() ;        }        if(reader.readLine(value) ==0){            return false ;        }        pos++ ;        return true ;    }}

MyFileInputFormat.java

package com.recordreader;import java.io.IOException;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.InputSplit;import org.apache.hadoop.mapreduce.JobContext;import org.apache.hadoop.mapreduce.RecordReader;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;public class MyFileInputFormat extends FileInputFormat<LongWritable, Text> {    @Override    public RecordReader<LongWritable, Text> createRecordReader(InputSplit arg0,            TaskAttemptContext arg1) throws IOException, InterruptedException {        return new MyRecordReader();    }    @Override    protected boolean isSplitable(JobContext context, Path filename) {        return false ;    }}

MyMapper.java

package com.recordreader;import java.io.IOException;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class MyMapper extends Mapper<LongWritable, Text, LongWritable, Text> {    @Override    protected void map(LongWritable key, Text value,Context context)            throws IOException, InterruptedException {        context.write(key, value) ;    }}

MyPartitioner.java

package com.recordreader;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Partitioner;public class MyPartitioner extends Partitioner<LongWritable, Text> {    @Override    public int getPartition(LongWritable key, Text value, int numPartitions) {        if(key.get() % 2 == 0){            key.set(1) ;            return 1 ;        }else{            key.set(0) ;            return 0 ;        }    }}

MyReducer.java

package com.recordreader;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;public class MyReducer extends Reducer<LongWritable, Text, Text, IntWritable> {    @Override    protected void reduce(LongWritable key, Iterable<Text> value,Context context)            throws IOException, InterruptedException {        int sum = 0 ;        for(Text val : value){            sum += Integer.valueOf(val.toString()) ;        }        Text writeKey = new Text() ;        IntWritable writeValue = new IntWritable() ;        if(key.get() == 0){            writeKey.set("奇数行之和：") ;        }else{            writeKey.set("偶数行之和：") ;        }        writeValue.set(sum) ;        context.write(writeKey, writeValue) ;    }}

TestRecordReader.java

package com.recordreader;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;import org.apache.hadoop.util.GenericOptionsParser;public class TestRecordReader {    public static void main(String args[]) throws Exception{        Configuration conf = new Configuration();        String[] otherArgs = new GenericOptionsParser(conf, args)                .getRemainingArgs();        if (otherArgs.length != 2) {            System.err.println("Usage: wordcount <in> <out>");            System.exit(2);        }        Job job = new Job(conf, "word count");        job.setJarByClass(TestRecordReader.class);        job.setMapperClass(MyMapper.class);        job.setReducerClass(MyReducer.class);        job.setPartitionerClass(MyPartitioner.class) ;        job.setNumReduceTasks(2) ;        job.setInputFormatClass(MyFileInputFormat.class) ;//      job.setOutputKeyClass(Text.class);//      job.setOutputValueClass(IntWritable.class);        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));        System.exit(job.waitForCompletion(true) ? 0 : 1);    }}