MapReduce分析共同好友

两次使用mapreduce
输入文件:
A:B,C,D,F,E,O
B:A,C,E,K
C:F,A,D,I
D:A,E,F,L
E:B,C,D,M,L
F:A,B,C,D,E,O,M
G:A,C,D,E,F
H:A,C,D,E,O
I:A,O
J:B,O
K:A,C,D
L:D,E,F
M:E,F,G
O:A,H,I,J

最终需求结果如：A–B: C E
思路分析：从结论出发,最后一步的reduce 的key应为
A–B: c A–B: e
由此，改形式应该为第一次mapreduce的输出结果

所以进行两次mapreduce即可

FriendsPre.java:

public class FriendsPre {    static public class FriendsMapper extends Mapper<LongWritable, Text, Text, Text> {        Text friendText = new Text();        Text usrText = new Text();        @Override        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {            //获取每一行的好友内容            String line = value.toString();            String[] spil1 = line.split(":");            //当前用户            String usr = spil1[0];            //用户的所有好友            String[] friends = spil1[1].split(",");            //以该用户为为value，好友为key,输出给map            for (String friend:friends                 ) {                friendText.set(friend);                usrText.set(usr);                context.write(friendText, usrText);            }        }    }    static public class FriendsReducer extends Reducer<Text, Text, Text, NullWritable> {       Text usrText2 = new Text();        @Override        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {            //将iterator转化为list,因为iterator没有get方法，无法操作下标            List<Text> usersList = new ArrayList<Text>();            for (Text usr : values) {                //Text usrText = new Text();                //Text usrText = new Text();                    Text usrText = new Text(usr);                usersList.add(usrText);            }            for (int i = 0; i < usersList.size() - 1; i++) {                for (int j = i + 1; j < usersList.size(); j++) {                    //生成关系字符串如:a--b:c                    usrText2.set(usersList.get(i).toString() + "--"+usersList.get(j).toString()+":"+key.toString());                    context.write(usrText2, NullWritable.get());                }            }        }    }    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        Job job = Job.getInstance(conf);        job.setJarByClass(FriendsPre.class);        job.setJar("/Users/inequality/IdeaProjects/mapr/out/artifacts/mapr_jar/mapr.jar");        //设置调用类        job.setMapperClass(FriendsMapper.class);        job.setReducerClass(FriendsReducer.class);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(NullWritable.class);        //指定输入输入输出路径        FileInputFormat.setInputPaths(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[1]));        FileSystem fs = FileSystem.get(conf);        if (fs.exists(new Path(args[1]))) {            fs.delete(new Path(args[1]),true);        }        //将job中配置的相关参数，以及job所用的java类所用的jar包，提交给yarn去运行        //job.submit();        boolean res = job.waitForCompletion(true);        System.exit(res ? 0 : 1);    }}

FirendsLast.java

    public class FriendsLast {    static class FriendsLastMapper extends Mapper<LongWritable, Text, Text, Text> {        Text relationText = new Text();        Text friendText = new Text();        @Override        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {            String line = value.toString();            String[] words = line.split(":");            //关系字符串            String relation = words[0];            //共同好友            String friend = words[1];            //输出            relationText.set(relation);            friendText.set(friend);            context.write(relationText, friendText);        }    }    static class FriendsLastReducer extends Reducer<Text, Text, Text, NullWritable> {        Text re = new Text();        @Override        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {            StringBuffer reSb = new StringBuffer(key.toString());            reSb.append(": ");//a--b:            for (Text friend : values) {                reSb.append(friend.toString()).append(" ");            }            re.set(reSb.toString());            context.write(re, NullWritable.get());        }    }    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        Job job = Job.getInstance(conf);        job.setJarByClass(FriendsPre.class);        job.setJar("/Users/inequality/IdeaProjects/mapr/out/artifacts/mapr_jar/mapr.jar");        //设置调用类        job.setMapperClass(FriendsLastMapper.class);        job.setReducerClass(FriendsLastReducer.class);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(NullWritable.class);        //指定输入输入输出路径        FileInputFormat.setInputPaths(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[1]));        FileSystem fs = FileSystem.get(conf);        if (fs.exists(new Path(args[1]))) {            fs.delete(new Path(args[1]),true);        }        //将job中配置的相关参数，以及job所用的java类所用的jar包，提交给yarn去运行        //job.submit();        boolean res = job.waitForCompletion(true);        System.exit(res ? 0 : 1);    }