mapreduce二次排序

来源:互联网 发布:吸引程序员读的书 编辑:程序博客网 时间:2024/05/15 09:23

现在想统计近年的最高气温和最低气温,按照最高气温做降序,如果最高温度一样,按照最低温度升序
package sort;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class SortApp {
 /**
  * 现在想统计近年的最高气温和最低气温,按照最高气温做降序,如果最高温度一样,按照最低温度升序
  * 2014  40  -20
  * 2013  38  -2
  * 2012  37   1
  * 2011  40  -18
  * 2010  35   3
  * 2009  38   3
  * 2008  35   2
  * ===========================================
  * 排序后结果应该为
  * 2014  40  -20
  * 2011  40  -18
  * 2013  38  -2
  * 2009  38   3
  * 2012  37   1
  * 2008  35   2
  * 2010  35   3
  * @param args
  * @throws URISyntaxException
  * @throws IOException
  * @throws ClassNotFoundException
  * @throws InterruptedException
  */
 public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException, ClassNotFoundException {
  // TODO Auto-generated method stub
  String INPUT_PATH = "hdfs://xiaoqi:9000/templog";
  String OUT_PATH = "hdfs://xiaoqi:9000/result";
  Configuration conf = new Configuration();
  Path path = new Path(OUT_PATH);
  FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
  if(fileSystem.exists(path)){
   fileSystem.delete(path);
  }
  Job job = new Job(conf, SortApp.class.getSimpleName());
  FileInputFormat.setInputPaths(job, INPUT_PATH);
  job.setInputFormatClass(TextInputFormat.class);
  job.setMapperClass(MyMapper.class);
  job.setMapOutputKeyClass(TempWritable.class);
  job.setMapOutputValueClass(LongWritable.class);
  job.setPartitionerClass(HashPartitioner.class);
  job.setNumReduceTasks(1);
  job.setReducerClass(MyReducer.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(TempWritable.class);
  FileOutputFormat.setOutputPath(job, path);
  job.setOutputFormatClass(TextOutputFormat.class);
  job.waitForCompletion(true);
  
 }
}

class MyMapper extends Mapper<LongWritable, Text, TempWritable, LongWritable>{
 protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,TempWritable,LongWritable>.Context context) throws java.io.IOException ,InterruptedException {
  String vs[] = value.toString().split("\t");
  long temmax = Long.parseLong(vs[1]);
  long temmin = Long.parseLong(vs[2]);
  TempWritable tempWritable = new TempWritable(temmax,temmin);
  context.write(tempWritable,new LongWritable(Long.parseLong(vs[0])));
 };
}
class MyReducer extends Reducer<TempWritable,LongWritable, LongWritable,TempWritable>{
 protected void reduce(TempWritable key, java.lang.Iterable<LongWritable> values, org.apache.hadoop.mapreduce.Reducer<TempWritable,LongWritable,LongWritable,TempWritable>.Context context) throws IOException ,InterruptedException {
  for (LongWritable year : values) {
   context.write(year, key);
  }
 };
}


class TempWritable implements WritableComparable{

 long maxtemp;
 long mintemp;
 public TempWritable(long maxtemp,long mintemp){
  this.maxtemp = maxtemp;
  this.mintemp = mintemp;
 }
 public TempWritable(){}
 
 @Override
 public void write(DataOutput out) throws IOException {
  out.writeLong(maxtemp);
  out.writeLong(mintemp);
 }

 @Override
 public void readFields(DataInput in) throws IOException {
  this.maxtemp = in.readLong();
  this.mintemp = in.readLong();
 }

 @Override
 public int compareTo(Object o) {
  // TODO Auto-generated method stub
  TempWritable temp = (TempWritable)o;
  long k = this.maxtemp - temp.maxtemp;
  if(k != 0){
   return -(int)k;
  }
  return (int)(this.mintemp - temp.mintemp);
 }
 
 @Override
 public int hashCode() {
  // TODO Auto-generated method stub
  return (int)this.maxtemp+(int)this.mintemp;
 }
 
 @Override
 public boolean equals(Object obj) {
  // TODO Auto-generated method stub
  if(!(obj instanceof TempWritable))
   return false;
  TempWritable temp = (TempWritable)obj;
  return (this.maxtemp == temp.maxtemp)&&(this.mintemp == temp.mintemp);
 }
 
 @Override
 public String toString() {
  // TODO Auto-generated method stub
  return maxtemp+"\t"+mintemp+"\t";
 }
 
}

 

 

 

 

 

 

 

0 0
原创粉丝点击