Mapreduce实例---共同好友

一：问题介绍

原始数据：每个人的好友列表

A:B,C,D,F,E,O

B:A,C,E,K

C:F,A,D,I

D:A,E,F,L

E:B,C,D,M,L

F:A,B,C,D,E,O,M

G:A,C,D,E,F

H:A,C,D,E,O

I:A,O

J:B,O

K:A,C,D

L:D,E,F

M:E,F,G

O:A,H,I,J

……

 

输出结果：每个人和其他各人所拥有的功能好友

A-B C,E,

A-C D,F,

A-D E,F,

A-E B,C,D,

A-F B,C,D,E,O,

A-G C,D,E,F,

A-H C,D,E,O,

A-I  O,

A-J  B,O,

A-K C,D,

A-L D,E,F,

A-M       E,F,

B-C A,

B-D A,E,

……

二：代码

步骤一：

public class CommonFriendsStepOne {public static class CommonFriendsStepOneMapper extends Mapper<LongWritable, Text, Text, Text>{private Text k = new Text();private Text v = new Text();// A:B,C,D,F,E,O       B--> A ; C-->A ; D-->A ; F-->A ....@Overrideprotected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException {String line = value.toString();String[] split = line.split(":");String[] friends = split[1].split(",");v.set(split[0]);for(String f : friends){k.set(f);context.write(k, v);}}}public static class CommonFriendsStepOneReducer extends Reducer<Text, Text, Text, Text>{/* * 输入的数据 <B A> <B E> <B F> <B J> * 输出的结果 <B A,E,F,J> */private Text v=new Text();@Overrideprotected void reduce(Text friend, Iterable<Text> persons,Context context)throws IOException, InterruptedException {StringBuilder sb=new StringBuilder();for(Text p : persons){sb.append(p).append(",");}v.set(sb.toString());context.write(friend, v);}}public static void main(String[] args) throws Exception, IOException {Configuration conf = new Configuration();Job job = Job.getInstance(conf);job.setJarByClass(CommonFriendsStepOne.class);job.setMapperClass(CommonFriendsStepOneMapper.class);job.setReducerClass(CommonFriendsStepOneReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);FileInputFormat.setInputPaths(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));boolean res = job.waitForCompletion(true);System.exit(res ? 0 : 1);}}

步骤二：

public class CommonFriendsStepTwo {static class CommonFriendsStepTwoMapper extends Mapper<LongWritable, Text, Text, Text>{//A I,K,C,B,G,F,H,O,D,       B A,F,J,E, private Text k=new Text();private Text v=new Text();@Overrideprotected void map(LongWritable key, Text value,Context context)throws IOException, InterruptedException {String line = value.toString();String[] split = line.split("\t");String[] persons = split[1].split(",");// 对用户进行排序，以免出现GF、FG被视为不同的组合Arrays.sort(persons);v.set(split[0]);// 对整个persons数组做两两组合拼接for(int i=0;i<persons.length-1;i++){for(int j=i+1;j<persons.length;j++){// 输出 <I-J A>k.set(persons[i] + "-" + persons[j]);context.write(k, v);}}}}static class CommonFriendsStepTwoReducer extends Reducer<Text, Text, Text, Text>{// <A-E,B> <A-E,C>.....private Text v = new Text();@Overrideprotected void reduce(Text pair, Iterable<Text> friends,Context context)throws IOException, InterruptedException {StringBuilder sb=new StringBuilder();for(Text f : friends){sb.append(f).append(" ");}// <A-E, B C ..>v.set(sb.toString());context.write(pair, v);}}public static void main(String[] args) throws Exception, IOException {Configuration conf = new Configuration();Job job = Job.getInstance(conf);job.setJarByClass(CommonFriendsStepTwo.class);job.setMapperClass(CommonFriendsStepTwoMapper.class);job.setReducerClass(CommonFriendsStepTwoReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);FileInputFormat.setInputPaths(job, new Path(args[0]));FileOutputFormat.setOutputPath(job, new Path(args[1]));boolean res = job.waitForCompletion(true);System.exit(res ? 0 : 1);}}