Hadoop自定义Writable实现二次排序

输入数据集

20,75,cqu
25,90,cqnu
20,70,cqupt
24,80,cquk

二次排序功能

先按第一列数字排序，再按第二列数字排序

输出结果

20,70 cqupt
20,75 cqu
24,80 cquk
25,90 cqnu

实现原理

因为MapReduce的输出是按key排序的，所以，我们可以自定义一个key，这个key包含第一列和第二列。在实现compareTo方法时，先按第一列排序，再按第二列排序。

实现代码

import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class TestJoin {    public static class IntPairWritable implements WritableComparable<IntPairWritable>{        private IntWritable first;        IntWritable second;        public IntPairWritable(){            set(new IntWritable(), new IntWritable());        }        public void set(IntWritable first, IntWritable second){            this.first = first;            this.second = second;        }        @Override        public void write(DataOutput out) throws IOException {            this.first.write(out);            this.second.write(out);        }        @Override        public void readFields(DataInput in) throws IOException {            this.first.readFields(in);            this.second.readFields(in);        }        @Override        public int compareTo(IntPairWritable o) {            int result = this.first.compareTo(o.first);            if(result != 0)                return result;            return this.second.compareTo(o.second);        }        public boolean equals(Object o){            if(o instanceof IntPairWritable){                IntPairWritable obj = (IntPairWritable)o;                return this.first.equals(obj.first) && this.second.equals(obj.second);            }            return false;        }        public String toString(){            return this.first.toString() + "," + this.second.toString();        }    }    public static class SecondSortMapper extends Mapper<LongWritable, Text , IntPairWritable , Text>{        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{            String array[] = value.toString().split(",");            IntPairWritable keyPair = new IntPairWritable();            keyPair.set(new IntWritable(Integer.valueOf(array[0])),new IntWritable( Integer.valueOf(array[1])));            context.write(keyPair, new Text(array[2]));        }    }    public static class TestJoinReducer extends Reducer<IntPairWritable, Text, IntPairWritable, Text>{        public void reduce(IntPairWritable key, Iterable<Text> value, Context context) throws IOException, InterruptedException{            for(Text v : value){                context.write(key, v);            }        }    }    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        if(args.length < 2){            System.out.println("args must be three");return ;        }        Configuration conf = new Configuration();        Job job = Job.getInstance(conf,"TestJoin");        job.setJarByClass(TestJoin.class);        job.setMapperClass(SecondSortMapper.class);        job.setReducerClass(TestJoinReducer.class);        job.setOutputKeyClass(IntPairWritable.class);  //        job.setOutputValueClass(Text.class);        FileInputFormat.addInputPath(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[2]));        System.exit(job.waitForCompletion(true)?0:1);    }}

注意事项

在自定义Writable时，要实现的方法有read，write，compareTo三个方法。
除此之外，还要重写toString方法，以便于key可以输出内容。