mapreduce二次排序
来源:互联网 发布:吸引程序员读的书 编辑:程序博客网 时间:2024/05/15 09:23
现在想统计近年的最高气温和最低气温,按照最高气温做降序,如果最高温度一样,按照最低温度升序
package sort;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
public class SortApp {
/**
* 现在想统计近年的最高气温和最低气温,按照最高气温做降序,如果最高温度一样,按照最低温度升序
* 2014 40 -20
* 2013 38 -2
* 2012 37 1
* 2011 40 -18
* 2010 35 3
* 2009 38 3
* 2008 35 2
* ===========================================
* 排序后结果应该为
* 2014 40 -20
* 2011 40 -18
* 2013 38 -2
* 2009 38 3
* 2012 37 1
* 2008 35 2
* 2010 35 3
* @param args
* @throws URISyntaxException
* @throws IOException
* @throws ClassNotFoundException
* @throws InterruptedException
*/
public static void main(String[] args) throws IOException, URISyntaxException, InterruptedException, ClassNotFoundException {
// TODO Auto-generated method stub
String INPUT_PATH = "hdfs://xiaoqi:9000/templog";
String OUT_PATH = "hdfs://xiaoqi:9000/result";
Configuration conf = new Configuration();
Path path = new Path(OUT_PATH);
FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
if(fileSystem.exists(path)){
fileSystem.delete(path);
}
Job job = new Job(conf, SortApp.class.getSimpleName());
FileInputFormat.setInputPaths(job, INPUT_PATH);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(TempWritable.class);
job.setMapOutputValueClass(LongWritable.class);
job.setPartitionerClass(HashPartitioner.class);
job.setNumReduceTasks(1);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(TempWritable.class);
FileOutputFormat.setOutputPath(job, path);
job.setOutputFormatClass(TextOutputFormat.class);
job.waitForCompletion(true);
}
}
class MyMapper extends Mapper<LongWritable, Text, TempWritable, LongWritable>{
protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,TempWritable,LongWritable>.Context context) throws java.io.IOException ,InterruptedException {
String vs[] = value.toString().split("\t");
long temmax = Long.parseLong(vs[1]);
long temmin = Long.parseLong(vs[2]);
TempWritable tempWritable = new TempWritable(temmax,temmin);
context.write(tempWritable,new LongWritable(Long.parseLong(vs[0])));
};
}
class MyReducer extends Reducer<TempWritable,LongWritable, LongWritable,TempWritable>{
protected void reduce(TempWritable key, java.lang.Iterable<LongWritable> values, org.apache.hadoop.mapreduce.Reducer<TempWritable,LongWritable,LongWritable,TempWritable>.Context context) throws IOException ,InterruptedException {
for (LongWritable year : values) {
context.write(year, key);
}
};
}
class TempWritable implements WritableComparable{
long maxtemp;
long mintemp;
public TempWritable(long maxtemp,long mintemp){
this.maxtemp = maxtemp;
this.mintemp = mintemp;
}
public TempWritable(){}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(maxtemp);
out.writeLong(mintemp);
}
@Override
public void readFields(DataInput in) throws IOException {
this.maxtemp = in.readLong();
this.mintemp = in.readLong();
}
@Override
public int compareTo(Object o) {
// TODO Auto-generated method stub
TempWritable temp = (TempWritable)o;
long k = this.maxtemp - temp.maxtemp;
if(k != 0){
return -(int)k;
}
return (int)(this.mintemp - temp.mintemp);
}
@Override
public int hashCode() {
// TODO Auto-generated method stub
return (int)this.maxtemp+(int)this.mintemp;
}
@Override
public boolean equals(Object obj) {
// TODO Auto-generated method stub
if(!(obj instanceof TempWritable))
return false;
TempWritable temp = (TempWritable)obj;
return (this.maxtemp == temp.maxtemp)&&(this.mintemp == temp.mintemp);
}
@Override
public String toString() {
// TODO Auto-generated method stub
return maxtemp+"\t"+mintemp+"\t";
}
}
- mapreduce二次排序
- MapReduce中二次排序
- MapReduce中的二次排序
- MapReduce之二次排序
- mapreduce二次排序
- MapReduce中的二次排序
- MapReduce二次排序。
- MapReduce中的二次排序
- MapReduce二次排序
- MapReduce 二次排序详解
- mapreduce的二次排序
- MapReduce二次排序
- MapReduce中的二次排序
- mapreduce 二次排序
- Hadoop MapReduce 二次排序
- mapreduce二次排序
- MapReduce二次排序
- MapReduce二次排序
- 面向对象的思想 泛型和委托
- Android Studio代理设置(SDK下载代理设置)
- linux内核读写文件
- vim 在所有行前插入相同的内容
- 浅谈软件测试流程
- mapreduce二次排序
- List of Unix utilities
- 如何使用UIAutomation进行iOS 自动化测试(一)
- MATLAB数据处理快速学习教程
- 随笔
- CC++初学者编程教程(3) 安装VS2010 boost标准库开发环境
- 小小记录
- Android 字体设置
- 今日小记