数据排序

样例输入

file1：

2

32

654

32

15

756

65223

file2：

5956

22

650

92

file3：

26

54

6

样例输出：

1 2

2  6

3 15

4 22

5 26

6 32

7 32

8 54

9 92

10 650

11 654

12 756

13 5956

14 65223

package mapreduce.test;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Partitioner;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class Sort {//map将输入中的value转化成IntWritable类型，作为输出的keypublic static class Map extends Mapper<Object, Text, IntWritable, IntWritable>{private static IntWritable data = new IntWritable();protected void map(Object key, Text value, Context context)throws IOException, InterruptedException {String line = value.toString();data.set(Integer.parseInt(line));context.write(data, new IntWritable(1));}}//reduce将输入的可以复制到输出的value上，然后根据输入的value-list中元素的个数决定key的输出次数//用全局的linenum来代表key的位次public static class Reduce extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable>{private static IntWritable linenum = new IntWritable(1);protected void reduce(IntWritable key, Iterable<IntWritable> values,Context context)throws IOException, InterruptedException {for(IntWritable val : values){context.write(linenum, key);linenum = new IntWritable(linenum.get()+1);}}}//自定义Partition函数，此函数根据输入数据的最大值和MapReduce框架中//Patition的数量获取将输入数据按照大小分块的边界，然后根据输入值和边界的关系返回对应的PartitionIDpublic static class Partition extends Partitioner<IntWritable, IntWritable>{@Overridepublic int getPartition(IntWritable key, IntWritable value,int numPartitions) {int Maxnumber = 65223;int bound = Maxnumber/numPartitions + 1;int keynumber = key.get();for(int i =0;i<numPartitions;i++){if(keynumber<bound*i && keynumber >= bound*(i-1))return i-1;}return -1;}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = new Job(conf,"sort");job.setJarByClass(Sort.class);job.setMapperClass(Map.class);job.setReducerClass(Reduce.class);job.setPartitionerClass(Partition.class);job.setOutputKeyClass(IntWritable.class);job.setOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));System.exit(job.waitForCompletion(true) ? 0 : 1);}}