Mapreduce算法四、key值和value值分别是组合值

来源：互联网发布：js身份证号码格式编辑：程序博客网时间：2024/05/17 08:29
package CartsDemo;import java.util.HashSet;import java.util.Set;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class TypeAndBrand {public static void main(String[] args) throws Exception {    if(args.length!=2){        System.exit(0);    }    Job job = new Job(new Configuration(),"TypeAndBrand");    job.setJarByClass(TypeAndBrand.class);    FileInputFormat.setInputPaths(job, new Path(args[0]));    FileOutputFormat.setOutputPath(job, new Path(args[1]));    job.setMapperClass(TypeAndBrandMap.class);    job.setReducerClass(TypeAndBrandReduce.class);    job.setMapOutputKeyClass(Text.class);    job.setMapOutputValueClass(Text.class);    job.setOutputKeyClass(Text.class);    job.setOutputValueClass(Text.class);    job.waitForCompletion(true);}public static class TypeAndBrandMap extends Mapper<LongWritable, Text, Text, Text>{    protected void map(LongWritable key, Text value, org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,Text,Text>.Context context) throws java.io.IOException ,InterruptedException {        String[] lines = value.toString().split("\t");        if(lines.length>7&&lines[7]!=null&&lines[8]!=null&&lines[7]!=""&&lines[8]!=""){            String brand = lines[7].trim();            String type = lines[8].trim();            if (lines.length>15&&lines[12]!=null&&lines[15]!=null&&lines[12]!=""&&lines[15]!="") {                String engine = lines[12].trim();                String oil = lines[15].trim();                context.write(new Text(type+"-"+brand), new Text(engine+"-"+oil));            }        }    };}public static class TypeAndBrandReduce extends Reducer<Text, Text, Text, Text>{    protected void reduce(Text key, java.lang.Iterable<Text> values, org.apache.hadoop.mapreduce.Reducer<Text,Text,Text,Text>.Context context) throws java.io.IOException ,InterruptedException {        Set<String > set = new HashSet<>();        for (Text value : values) {            set.add(value.toString());        }        context.write(key, new Text(set.toString()));    };}}
0 0