MapReduce典型编程场景——TopN算法自定义GroupComparator

1、需求

在统计学生成绩的小项目中，现有一个需求：
求出每个班参考学生成绩最高的学生的信息，班级、姓名、平均分

2、分析

利用“班级和平均分”作为key，可以将map阶段读取到的所有学生成绩数据按照班级和成绩排倒叙，发送到reduce
在reduce端利用GroupingComparator将班级相同的kv聚合成组，然后取第一个即是最大值

3、实现

第一步：先把分组和排序字段都综合到一个自定义对象里

import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import org.apache.hadoop.io.WritableComparable;public class ClazzScore implements WritableComparable{       private String clazz;       private Double score;       public String getClazz() {                return clazz;       }       public void setClazz(String clazz) {                this.clazz = clazz;       }       public Double getScore() {                return score;       }       public void setScore(Double score) {                this.score = score;       }       public ClazzScore(String clazz, Double score) {                 super();                 this.clazz = clazz;                 this.score = score;        }        public ClazzScore() {                  super();        }        @Override        public String toString() {                  return clazz + "\t" + score;        }         @Override       // 序列化        public void write(DataOutput out) throws IOException {                   out.writeUTF(clazz);                   out.writeDouble(score);        }         @Override       // 反序列化        public void readFields(DataInput in) throws IOException {                     this.clazz = in.readUTF();                     this.score = in.readDouble();        }        /**         * key 排序         */         @Override         public int compareTo(ClazzScore cs) {                     int it = cs.getClazz().compareTo(this.clazz);                     if(it == 0){                            return (int) (cs.getScore() - this.score);                     }else{                            return it;                     }           }}

第二步：编写排序之后的ClazzScore数据传入ReduceTask的分组规则

import org.apache.hadoop.io.WritableComparable;import org.apache.hadoop.io.WritableComparator;public class ClazzScoreGroupComparator extends WritableComparator{       ClazzScoreGroupComparator(){                    super(ClazzScore.class, true);        }        /**         * 决定输入到 reduce 的数据的分组规则         */        @Override        public int compare(WritableComparable a, WritableComparable b) {                       ClazzScore cs1 = (ClazzScore)a;                       ClazzScore cs2 = (ClazzScore)b;                       int it = cs1.getClazz().compareTo(cs2.getClazz());                       return it;         }}

第三步：编写MapReduce程序

import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.DoubleWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;     /**      * TopN 问题      */ public class ScoreTop1MR {         public static void main(String[] args) throws Exception {                 Configuration conf = new Configuration();                  Job job = Job.getInstance(conf);                   job.setJarByClass(ScoreTop1MR.class);                   job.setMapperClass(ScoreTop1MRMapper.class);                   job.setReducerClass(ScoreTop1MRReducer.class);                   job.setOutputKeyClass(ClazzScore.class);                   job.setOutputValueClass(DoubleWritable.class);                  // 设置传入 reducer 的数据分组规则                  job.setGroupingComparatorClass(ClazzScoreGroupComparator.class);                 FileInputFormat.setInputPaths(job, "d:/score_all/input");                 Path p = new Path("d:/score_all/output1");                 FileSystem fs = FileSystem.newInstance(conf);                 if(fs.exists(p)){                            fs.delete(p, true);                 }                 FileOutputFormat.setOutputPath(job, p);                 boolean status = job.waitForCompletion(true);                 System.exit(status ? 0 : 1);           }         static class ScoreTop1MRMapper extends Mapper{                @Override                 protected void map(LongWritable key, Text value, Context context) throws IOException,InterruptedException {                       String[] splits = value.toString().split("\t");                       ClazzScore cs = new ClazzScore(splits[0], Double.parseDouble(splits[2]));                       context.write(cs, new DoubleWritable(Double.parseDouble(splits[2])));                 }           }          static class ScoreTop1MRReducer extends Reducer{                 @Override                 protected void reduce(ClazzScore cs, Iterable scores, Context  context) throws IOException, InterruptedException {                             // 按照规则，取每组的第一个就是 Top1                 context.write(cs, scores.iterator().next());                  }          }}