hadoop 求key最大值

来源：互联网发布：php项目管理系统源码编辑：程序博客网时间：2024/06/06 17:46

那么，在之前，我们是否考虑到了一个问题：什么时候才可以打印出最大值？在所有的map任务结束的时候，就调用cleanup(...)时，

略说一下，map与reduce之间传值的原理：

在源文件将一行数据，分解成<key,value>时，会统一由map加工（调用map()），输出新的<key,value>，然后进行分区，在分区里面进行排序，然后输出给reduce

static class MyMapper extends Mapper<LongWritable, Text, LongWritable, NullWritable>{long max = Long.MIN_VALUE;protected void map(LongWritable k1, Text v1, Context context) throws java.io.IOException ,InterruptedException {final long temp = Long.parseLong(v1.toString());if(temp>max){max = temp;}};protected void cleanup(org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,LongWritable, NullWritable>.Context context) throws java.io.IOException ,InterruptedException {context.write(new LongWritable(max), NullWritable.get());};}static class MyReducer extends Reducer<LongWritable, NullWritable, LongWritable, NullWritable>{long max = Long.MIN_VALUE;protected void reduce(LongWritable k2, java.lang.Iterable<NullWritable> arg1, org.apache.hadoop.mapreduce.Reducer<LongWritable,NullWritable,LongWritable,NullWritable>.Context arg2) throws java.io.IOException ,InterruptedException {final long temp = k2.get();if(temp>max){max = temp;}};protected void cleanup(org.apache.hadoop.mapreduce.Reducer<LongWritable,NullWritable,LongWritable,NullWritable>.Context context) throws java.io.IOException ,InterruptedException {context.write(new LongWritable(max), NullWritable.get());};}

0 0