Hadoop中自定义排序，分区，分组

–Reducer

import java.io.IOException;    import org.apache.hadoop.io.Text;  import org.apache.hadoop.mapreduce.Reducer;  public class HotReduce extends Reducer<KeyPair, Text, KeyPair, Text>{           @Override          protected void reduce(KeyPair kp, Iterable<Text> i,Context context)                          throws IOException, InterruptedException {                  for (Text text : i) {                          context.write(kp, text);                  }          }           }  

–Mapper

import java.io.IOException;  import java.text.ParseException;  import java.text.SimpleDateFormat;  import java.util.Calendar;  import java.util.Date;    import org.apache.hadoop.io.LongWritable;  import org.apache.hadoop.io.Text;  import org.apache.hadoop.mapreduce.Mapper;  public class HotMapper extends Mapper<LongWritable, Text, KeyPair, Text>{          @Override          protected void map(LongWritable key, Text value,Context context)                          throws IOException, InterruptedException {                  String line=value.toString();                  String[] ss=line.split("\t");                  if (ss.length==2){                          int year=Integer.parseInt(ss[0].substring(0, 4));                          int hot=Integer.parseInt(ss[1].substring(0, ss[1].indexOf("°C")));                          KeyPair kp=new KeyPair();                          kp.setYear(year);                          kp.setHot(hot);                          context.write(kp, value);                  }          }  }  

–KeyPair 自定义封装类

import java.io.DataInput;  import java.io.DataOutput;  import java.io.IOException;  import org.apache.hadoop.io.WritableComparable;  public class KeyPair implements WritableComparable<KeyPair> {          private int year;          private int hot;          public int getYear() {                  return year;          }          public void setYear(int year) {                  this.year = year;          }          public int getHot() {                  return hot;          }          public void setHot(int hot) {                  this.hot = hot;          }          @Override  反序列化        public void readFields(DataInput in) throws IOException {                  this.year=in.readInt();                  this.hot=in.readInt();          }          @Override  序列化        public void write(DataOutput out) throws IOException {                  out.writeInt(year);                  out.writeInt(hot);          }          @Override  对比        public int compareTo(KeyPair o) {                                   int result=Integer.compare(year, o.getYear());                  if (result!=0){                          return result;                  }                  return Integer.compare(hot, o.hot);          }          @Override          public String toString() {                  return year+"\t"+hot;          }          @Override          public int hashCode() {                  return new Integer(year+hot).hashCode();          }  } 

–HotPartition
//自定义分区

import org.apache.hadoop.io.Text;  import org.apache.hadoop.mapreduce.Partitioner;  public class HotPartition extends Partitioner<KeyPair, Text>{          @Override          public int getPartition(KeyPair key, Text value, int num) {                  return (key.getYear()*127%num);          }  }  

–HotSort 自定义排序

import org.apache.hadoop.io.WritableComparable;  import org.apache.hadoop.io.WritableComparator;  public class HotSort extends WritableComparator{          public HotSort() {                  super(KeyPair.class, true);  //排序时使用自定义分装的类进行排序        }          @Override          public int compare(WritableComparable a, WritableComparable b) {                  KeyPair o1=(KeyPair) a;                  KeyPair o2=(KeyPair) b;                  int res=Integer.compare(o1.getYear(), o2.getYear());                  if (res!=0){                          return res;                  }                  return -Integer.compare(o1.getHot(),o2.getHot());//降序排序          }  }  

–HotGroup
自定义分组

import org.apache.hadoop.io.WritableComparable;  import org.apache.hadoop.io.WritableComparator;  public class HotGroup extends WritableComparator{          public HotGroup() {                  super(KeyPair.class, true);          }          @Override          public int compare(WritableComparable a, WritableComparable b) {                  KeyPair o1=(KeyPair) a;                  KeyPair o2=(KeyPair) b;                  return Integer.compare(o1.getYear(),o2.getYear());          }  } public class Job{    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        Configuration configuration = new Configuration();        Job job = new Job(configuration,"partation");        job.setJarByClass(Job.class);        job.setMapperClass(HotMapper.class);        job.setReducerClass(HotReducer.class);        job.setOutputKeyClass(KeyPair.class);        job.setOutputValueClass(Text.class);        job.setGroupingComparatorClass(FirstPartation.class);        job.setNumReduceTasks(3);        job.setSortComparatorClass(SortHot.class);        job.setGroupingComparatorClass(GroupHot.class);        FileInputFormat.addInputPath(job, new Path("/user/root/books"));        FileOutputFormat.setOutputPath(job, new Path("/user/root/bookout"));        System.exit(job.waitForCompletion(true)?0:1);    }}