MapReduce程序运行中的排序问题

来源：互联网发布：linux 获取文件行数编辑：程序博客网时间：2024/06/16 13:32

2016年4月5日19:43:27

MapReduce程序开发中的排序问题

作者：数据分析玩家

在MapReduce程序执行的第四步骤：对不同分区的数据，按照key进行排序、分组。在这里我们讲一下如何在MapReduce程序中进行自定义排序。

为了更好的说明问题，在文本博客中引入两个实例进行对比。先讲述第一个实例，这个实例我们并没有进行自定义排序，输入文件是sortdata.txt，内容如下：

3 3

3 2

3 1

2 2

2 1

1 1

我们期望讲过Mapreduce程序处理之后，输出的结果为：

1 1
2 1
2 2
3 1
3 2
3 3

即输出的结果要求：第一列升序排序，当第一列相同时，第二列进行升序排序。现在将我们没有自定义排序的源代码以及输出的结果列出：

package Sort;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;public class Sort0{        public static String path1 = "hdfs://hadoop:9000/sortdata.txt";        public static String path2 = "hdfs://hadoop:9000/sortdir";    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException        {          FileSystem fileSystem = FileSystem.get(new URI(path1), new Configuration());          if(fileSystem.exists(new Path(path2)))//输出路径事先是不能存在的          {            fileSystem.delete(new Path(path2), true);          }      Job job = new Job(new Configuration(),"Sort");          //编写MapReduce的运行步骤      FileInputFormat.setInputPaths(job, new Path(path1));      job.setInputFormatClass(TextInputFormat.class);      job.setMapperClass(MyMapper.class);      job.setNumReduceTasks(1);      job.setPartitionerClass(HashPartitioner.class);      job.setReducerClass(MyReducer.class);      job.setOutputKeyClass(LongWritable.class);      job.setOutputValueClass(LongWritable.class);      FileOutputFormat.setOutputPath(job, new Path(path2));      //提交任务      job.waitForCompletion(true);      //查看结果      FSDataInputStream fr = fileSystem.open(new Path("hdfs://hadoop:9000/sortdir/part-r-00000"));      IOUtils.copyBytes(fr, System.out, 1024, true);        }        public static class MyMapper extends Mapper<LongWritable, Text, LongWritable, LongWritable>        {         protected void map(LongWritable k1, Text v1,Context context)throws IOException, InterruptedException         {          String[] splited = v1.toString().split("\t");          String num1 = splited[0];          String num2 = splited[1];          context.write(new LongWritable(Long.parseLong(num1)),new LongWritable(Long.parseLong(num2)));         }        }        public static class MyReducer extends Reducer<LongWritable, LongWritable, LongWritable, LongWritable>        {        protected void reduce(LongWritable k2,Iterable<LongWritable> v2s,Context context)throws IOException, InterruptedException        {           for (LongWritable v2 : v2s){context.write(k2,v2);}        }        }}

MapReduce程序运行完之后输出的结果为：

1 1
2 2
2 1
3 3
3 2
3 1

从程序的运行结果我们可以看出，默认的key是按照升序进行输出的，但是value是不进行排序的。也就是在我们的shuffle阶段，经过了分区、排序之后，仅仅是对key进行排序，value是不参与排序的，下面引出文本的核心内容：

在Mapper阶段与Reducer阶段，参与排序的是key，value是不参与排序的，是不能进行排序的。

但是问题来了，如果我们想让value也参与排序，该如何处理呢，正如我们上面的例子一样，如何对value进行排序呢？

答案是：

自定义一个新的K2类,在这个K2类中含有原来的K2和V2,此时原来的K2和V2将作为一个新的K2存在,这时就可以排序了。

下面我们将自定义排序的MapReduce程序列出：

<pre name="code" class="java">package Sort;import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;public class Sort1{    public static String path1 = "hdfs://hadoop:9000/sortdata.txt";        public static String path2 = "hdfs://hadoop:9000/sortdir2";    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException       {        FileSystem fileSystem = FileSystem.get(new URI(path1), new Configuration());        if(fileSystem.exists(new Path(path2)))//输出路径事先是不能存在的        {          fileSystem.delete(new Path(path2), true);        }      Job job = new Job(new Configuration(),"Sort1");            //编写MapReduce的运行步骤      FileInputFormat.setInputPaths(job, new Path(path1));      job.setInputFormatClass(TextInputFormat.class);      job.setMapperClass(MyMapper.class);      job.setMapOutputKeyClass(SortWritable.class);      job.setMapOutputValueClass(NullWritable.class);      job.setNumReduceTasks(1);      job.setPartitionerClass(HashPartitioner.class);      job.setReducerClass(MyReducer.class);            job.setOutputKeyClass(SortWritable.class);      job.setOutputValueClass(NullWritable.class);            FileOutputFormat.setOutputPath(job, new Path(path2));      //提交任务      job.waitForCompletion(true);      //查看结果      FSDataInputStream fr = fileSystem.open(new Path("hdfs://hadoop:9000/sortdir2/part-r-00000"));      IOUtils.copyBytes(fr, System.out, 1024, true);       }         public static class MyMapper extends Mapper<LongWritable, Text, SortWritable, NullWritable>         {        protected void map(LongWritable k1, Text v1,Context context)throws IOException, InterruptedException        {                    String[] splited = v1.toString().split("\t");                    String num1 = splited[0];                    String num2 = splited[1];                    SortWritable vector = new SortWritable(Long.parseLong(num1),Long.parseLong(num2));                    context.write(vector, NullWritable.get());        }         }         public static class MyReducer extends Reducer<SortWritable, NullWritable, SortWritable, NullWritable>         {            protected void reduce(SortWritable k2,Iterable<NullWritable> v2s,Context context)throws IOException, InterruptedException            {               for (NullWritable v2 : v2s){context.write(k2,NullWritable.get());}            }              }}class SortWritable implements WritableComparable{            Long num1;            Long num2;            public SortWritable(){}            public SortWritable(long num1,long num2)            {            this.num1 = num1;            this.num2 = num2;            }public void write(DataOutput fw) throws IOException//序列化{                    fw.writeLong(num1);                    fw.writeLong(num2);}public void readFields(DataInput fr) throws IOException{                   this.num1 = fr.readLong();                   this.num2 = fr.readLong();}public int compareTo(Object obj){                    SortWritable cc = (SortWritable)obj;                    if(this.num1!=cc.num1)                    return (int) (this.num1 - cc.num1);//num1升序排列                    else                    return (int) (this.num2 - cc.num2);//num2升序排列                    }public String toString(){ return this.num1 + "\t" +this.num2;} }

正如程序所示：SortWritable是自定义排序的类，这个类当中包含了k2与v2，这样在shuffle阶段v2就可以参加排序了。

经过自定义排序，输出的结果为：

1   1
2   1
2   2
3   1
3   2
3 3

程序的运行结果正是我们想要的！

2016年4月5日20:18:27

0 0