Hadoop入门之共同好友实现Demo

以下是qq的好友列表数据，冒号前是一个用，冒号后是该用户的所有好友（数据中的好友关系是单向的）

A:B,C,D,F,E,O

B:A,C,E,K

C:F,A,D,I

D:A,E,F,L

E:B,C,D,M,L

F:A,B,C,D,E,O,M

G:A,C,D,E,F

H:A,C,D,E,O

I:A,O

J:B,O

K:A,C,D

L:D,E,F

M:E,F,G

O:A,H,I,J

 

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

package com.demo.friends;import java.io.IOException;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;/** * @Description:  *  1.以key为聚合点 将某个好友点先聚合出来 ,且对value集合进行排序 *    B:A,E,F,J *    C:A,B,E,F,G,H,K *  2.以排序后的两个人为key,再次进行聚合操作 *    A-E:B,C *    .... *     * @author: songqinghu * @date: 2017年8月30日 下午5:16:49 * Version:1.0 */public class FriendOneStep {        public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        conf.set("HADOOP_USER_NAME", "hadoop");        Job job = Job.getInstance(conf);                job.setJarByClass(FriendOneStep.class);                job.setMapperClass(FriendOneStepMapper.class);        job.setReducerClass(FriendOneStepReducer.class);                job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(Text.class);                        FileInputFormat.setInputPaths(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[1]));                //        job.setNumReduceTasks(2);                job.waitForCompletion(true);    }}class FriendOneStepReducer extends Reducer<Text, Text, Text, Text>{        @Override    protected void reduce(Text text, Iterable<Text> iters, Context context)            throws IOException, InterruptedException {         //收集后进行排序组装        List<String> list = new ArrayList<String>();        for (Text t : iters) {            list.add(t.toString());        }        Object[] array = list.toArray();        Arrays.sort(array);                StringBuffer sb = new StringBuffer();        for (int i = 0; i < array.length; i++) {            sb.append(array[i].toString());            if(i<array.length-1){                sb.append(",");            }        }                context.write(text, new Text(sb.toString()));            }    }class FriendOneStepMapper extends Mapper<LongWritable, Text, Text, Text>{        Text outKey = new Text();    Text outValue = new Text();        @Override    protected void map(LongWritable lon, Text value, Context context)            throws IOException, InterruptedException {                String line = value.toString();        String[] temp = line.split(":");        outValue.set(temp[0]);                String[] keys = temp[1].split(",");                for (String key : keys) {            outKey.set(key);            context.write(outKey, outValue);        }    }    }

package com.demo.friends;import java.io.IOException;import java.util.Iterator;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;/** *  * @Description: 这一步以两个用户为Key和聚合 * @author: songqinghu * @date: 2017年8月30日 下午5:52:50 * Version:1.0 */public class FriendTwoStep {    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        conf.set("HADOOP_USER_NAME", "hadoop");        Job job = Job.getInstance(conf);                job.setJarByClass(FriendTwoStep.class);                job.setMapperClass(FriendTwoStepMapper.class);        job.setReducerClass(FriendTwoStepReducer.class);                job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(NullWritable.class);                        FileInputFormat.setInputPaths(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[1]));                //        job.setNumReduceTasks(2);                job.waitForCompletion(true);    }        }class FriendTwoStepReducer extends Reducer<Text, Text, Text,NullWritable>{        @Override    protected void reduce(Text text, Iterable<Text> iters, Context context)            throws IOException, InterruptedException {        //相同的键值对聚合在一起,将其共同好友组合在一起                StringBuffer sb = new StringBuffer();        sb.append(text.toString()).append(":");        Iterator<Text> iterator = iters.iterator();        while (iterator.hasNext()) {            sb.append(iterator.next().toString());            if(iterator.hasNext()){                sb.append(",");            }                    }        context.write(new Text(sb.toString()), null);    }    }class FriendTwoStepMapper extends Mapper<LongWritable, Text, Text, Text>{        Text outKey = new Text();    Text outValue = new Text();    @Override    protected void map(LongWritable key, Text value, Context context)            throws IOException, InterruptedException {        String line = value.toString();        String[] temp = line.split("\t");                outValue.set(temp[0]);                //切割出各个用户        String[] friends = temp[1].split(",");        //如果数量小于2那就跳过        if(friends.length>1){            //组装key            for (int i = 0; i < friends.length-1; i++) {                                for (int j = i+1; j < friends.length; j++) {                    outKey.set(friends[i]+"-"+friends[j]);                    context.write(outKey,outValue);                }            }                    }            }    }