算法(2) TopN Mapreduce/Spark

原理是每个map保留TopN数据，然后发送给reduce, reduce只设置一个，也就是说如果有10个map, 那么每个map处理好数据保留TopN列表，然后10个map就是100条数据，然后发送给reduce. 然后由reduce来过滤出10条数据。

每个Mapper开始之前有setup方法，结束之后有cleanup， 通常不会使用，setup, cleanup仅仅是每个mapper开始或者结束后才调用，仅仅调用一次。 上述算法原理是map来过滤出数据，然后统一交个cleanup发送给reduce.

reduce层原理也一样，假设10个map数据传给reduce,那么会有reduce会接受top100条数据，然后交给cleanup一次性过滤，然后output.  这也是为什么仅仅设置一个reduce的原因。

原始数据如下：

30;"unemployed";"married";"primary";"no";1787;"no";"no";"cellular";19;"oct";79;1;-1;0;"unknown";"no"33;"services";"married";"secondary";"no";4789;"yes";"yes";"cellular";11;"may";220;1;339;4;"failure";"no"35;"management";"single";"tertiary";"no";1350;"yes";"no";"cellular";16;"apr";185;1;330;1;"failure";"no"30;"management";"married";"tertiary";"no";1476;"yes";"yes";"unknown";3;"jun";199;4;-1;0;"unknown";"no"59;"blue-collar";"married";"secondary";"no";0;"yes";"no";"unknown";5;"may";226;1;-1;0;"unknown";"no"35;"management";"single";"tertiary";"no";747;"no";"no";"cellular";23;"feb";141;2;176;3;"failure";"no"36;"self-employed";"married";"tertiary";"no";307;"yes";"no";"cellular";14;"may";341;1;330;2;"other";"no"39;"technician";"married";"secondary";"no";147;"yes";"no";"cellular";6;"may";151;2;-1;0;"unknown";"no"41;"entrepreneur";"married";"tertiary";"no";221;"yes";"no";"unknown";14;"may";57;2;-1;0;"unknown";"no"43;"services";"married";"primary";"no";-88;"yes";"yes";"cellular";17;"apr";313;1;147;2;"failure";"no"39;"services";"married";"secondary";"no";9374;"yes";"no";"unknown";20;"may";273;1;-1;0;"unknown";"no"43;"admin.";"married";"secondary";"no";264;"yes";"no";"cellular";17;"apr";113;2;-1;0;"unknown";"no"36;"technician";"married";"tertiary";"no";1109;"no";"no";"cellular";13;"aug";328;2;-1;0;"unknown";"no"20;"student";"single";"secondary";"no";502;"no";"no";"cellular";30;"apr";261;1;-1;0;"unknown";"yes"31;"blue-collar";"married";"secondary";"no";360;"yes";"yes";"cellular";29;"jan";89;1;241;1;"failure";"no"40;"management";"married";"tertiary";"no";194;"no";"yes";"cellular";29;"aug";189;2;-1;0;"unknown";"no"56;"technician";"married";"secondary";"no";4073;"no";"no";"cellular";27;"aug";239;5;-1;0;"unknown";"no"37;"admin.";"single";"tertiary";"no";2317;"yes";"no";"cellular";20;"apr";114;1;152;2;"failure";"no"25;"blue-collar";"single";"primary";"no";-221;"yes";"no";"unknown";23;"may";250;1;-1;0;"unknown";"no"31;"services";"married";"secondary";"no";132;"no";"no";"cellular";7;"jul";148;1;152;1;"other";"no"38;"management";"divorced";"unknown";"no";0;"yes";"no";"cellular";18;"nov";96;2;-1;0;"unknown";"no"42;"management";"divorced";"tertiary";"no";16;"no";"no";"cellular";19;"nov";140;3;-1;0;"unknown";"no"44;"services";"single";"secondary";"no";106;"no";"no";"unknown";12;"jun";109;2;-1;0;"unknown";"no"

.............................. 上面只是数据样例，实际还有很多行.

实现代码如下：

package com.isesol.mapreduce;import java.io.IOException;import java.sql.Connection;import java.sql.DriverManager;import java.sql.SQLException;import java.sql.Statement;import java.util.ArrayList;import java.util.Iterator;import java.util.Map;import java.util.SortedMap;import java.util.StringTokenizer;import java.util.TreeMap;import java.util.TreeSet;import org.apache.commons.net.nntp.NewsgroupInfo;import org.apache.hadoop.conf.Configurable;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.TextInputFormat;import org.apache.hadoop.mapred.TextOutputFormat;import org.apache.hadoop.mapred.lib.NullOutputFormat;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.JobContext;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.OutputCommitter;import org.apache.hadoop.mapreduce.OutputFormat;import org.apache.hadoop.mapreduce.Partitioner;import org.apache.hadoop.mapreduce.RecordWriter;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.Reducer.Context;import org.apache.hadoop.mapreduce.TaskAttemptContext;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.SplitLineReader;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;public class topN {public static class TokenizerMapper extends Mapper<Object, Text, Text, Text> {private SortedMap top10 = new TreeMap();public void map(Object key, Text value, Context context) throws IOException, InterruptedException {String str = "";String[] val = value.toString().split(";");for (int i = 1; i < val.length - 1; i++) {str += val[i];}top10.put(val[0], str);if (top10.size() > 10) {top10.remove(top10.firstKey());}}public void cleanup(Context context) throws IOException, InterruptedException {Iterator iterator = top10.entrySet().iterator();while (iterator.hasNext()) {Map.Entry ent = (Map.Entry) iterator.next();String key = ent.getKey().toString();String value = ent.getValue().toString();context.write(new Text(key), new Text(value));}}}public static class IntSumReducer extends Reducer<Text, Text, Text, Text> {private SortedMap top10 = new TreeMap();public void reduce(Text key, Iterable<Text> value, Context context) throws IOException, InterruptedException {for( Text val : value) {top10.put(key.toString(), val.toString());}}public void cleanup(Context context) throws IOException, InterruptedException {System.out.println("top10 size isa: " + top10.size());while(top10.size() > 10) {top10.remove(top10.firstKey());}Iterator iterator = top10.entrySet().iterator();System.out.println("top10 size is: " + top10.size());while (iterator.hasNext()) {Map.Entry ent = (Map.Entry) iterator.next();String key01 = ent.getKey().toString();String value01 = ent.getValue().toString();context.write(new Text(key01), new Text(value01));} }}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = Job.getInstance(conf, "topN");job.setJarByClass(topN.class);job.setMapperClass(TokenizerMapper.class);job.setReducerClass(IntSumReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);job.setNumReduceTasks(1);// job.setPartitionerClass(twopartitions.class);// job.setOutputFormatClass(fakeOutPutFormat.class);FileInputFormat.addInputPath(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));System.exit(job.waitForCompletion(true) ? 0 : 1);}}

结果如下：

76"retired""married""primary""no"2590"no""no""telephone"9"feb"6812-10"unknown"77"retired""married""primary""no"680"no""no""telephone"27"nov"3414943"failure"78"retired""married""tertiary""no"226"no""no""telephone"6"nov"1361-10"unknown"79"retired""divorced""unknown""no"2628"no""no""telephone"8"jul"22074502"failure"80"housemaid""married""primary""no"0"no""no""cellular"23"feb"63911891"failure"81"retired""married""secondary""no"1"no""no""cellular"19"aug"655-10"unknown"83"retired""divorced""primary""no"1097"no""no""telephone"5"mar"1811-10"unknown"84"retired""divorced""primary""no"639"no""no""telephone"18"may"3533-10"unknown"86"retired""married""secondary""no"1503"no""no""telephone"18"mar"16531011"other"87"retired""married""primary""no"230"no""no""cellular"30"oct"1441-10"unknown"

上面没有考虑有重复数据，如果需要考虑重复数据也很好处理，稍微加工一下即可。

上面的TOP N是固定的，通常在实际些程序不会把N固定，比如我想TOP 100，不可能再去修改参数，因此赋予变量是最好的方式。Mpreduce通过conf.set(name, value)的方式设置变量，然后通过get获取变量。

conf.set("topn", args[2]);

topn = context.getConfiguration().get("topn");if(topn == null) {topn = "10";}