第四个MapReduce程序----两表join

来源：互联网发布：tiny core linux 编辑：程序博客网时间：2024/05/16 14:29
需求：两个表，一个有id和name，一个有id和phoneNum，得到name和与之对应的phoneNum

package club.drguo.mapreduce.joinquery;import java.io.IOException;import java.util.ArrayList;import org.apache.commons.lang.StringUtils;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.FileSplit;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class JoinQuery {public static class JoinQueryMapper extends Mapper<LongWritable, Text, Text, Text> {@Overrideprotected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)throws IOException, InterruptedException {String line = value.toString();String[] strings = StringUtils.split(line, " ");String id = strings[0];String phoneNum = strings[1];FileSplit fileSplit = (FileSplit) context.getInputSplit();String filename = fileSplit.getPath().getName();context.write(new Text(id), new Text(phoneNum + "---" + filename));// K：001// V：123---a.txt}}public static class JoinQueryReducer extends Reducer<Text, Text, Text, Text> {// K：001 V：{123---a.txt,124---b.txt}@Overrideprotected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)throws IOException, InterruptedException {// 第一次循环拿出a表中的那个字段String leftKey = "";ArrayList<String> rightFields = new ArrayList<>();for (Text value : values) {if (value.toString().contains("a.txt")) {leftKey = StringUtils.split(value.toString(), "---")[0];} else {rightFields.add(value.toString());}}// 再用leftkey去遍历拼接b表中的字段，并输出结果for (String field : rightFields) {String result = "";result += leftKey + "\t" + StringUtils.split(field.toString(), "---")[0];context.write(new Text(leftKey), new Text(result));}}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job joinjob = Job.getInstance(conf);joinjob.setJarByClass(JoinQuery.class);joinjob.setMapperClass(JoinQueryMapper.class);joinjob.setReducerClass(JoinQueryReducer.class);joinjob.setOutputKeyClass(Text.class);joinjob.setOutputValueClass(Text.class);FileInputFormat.setInputPaths(joinjob, new Path(args[0]));FileOutputFormat.setOutputPath(joinjob, new Path(args[1]));joinjob.waitForCompletion(true);}}
0 0