mapReduce使用分布式缓存机制

来源：互联网发布：c语言中的error 编辑：程序博客网时间：2024/06/06 18:42

由于有时候reduce value值数量的不确定性可能会造成数据倾斜,可以考虑使用分布式缓存机制，仅用map进行输出。

mapreduce中具体的函数为

job.addCacheFile(new URI("file:///Users/inequality/tmp/input/join/pro.txt"));

此函数可以在map运行之前在工作目录加入缓存文件，供map使用

具体程序代码：实现采购信息和商品信息的map端合并

采购表
1001,20150710,1,2
1002,20100910,1,3
1002,20120912,2,3

商品信息
1,xiaomi
2,chuizi

运行结果
1001,20150710,1,2 mi6
1002,20100910,1,3 mi6
1002,20120912,2,3 chuizi

public class MapJoin {    static class MapJoinMapper extends Mapper<LongWritable, Text, Text, NullWritable> {        Map<String, String> pfInfoMap = new HashMap<String,String>();        Text reText = new Text();        @Override        protected void setup(Context context) throws IOException, InterruptedException {            BufferedReader bf = new BufferedReader(new InputStreamReader(new FileInputStream("pro.txt")));            String line;            while (StringUtils.isNotEmpty(line = bf.readLine())) {                String[] fields = line.split(",");                pfInfoMap.put(fields[0], fields[1]);            }            bf.close();        }        @Override        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {            String orderLine = value.toString();            String orderWords[] = orderLine.split(",");            String pid = orderWords[2];            reText.set(orderLine + "\t" + pfInfoMap.get(pid));            context.write(reText, NullWritable.get());        }    }    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        Job job = Job.getInstance(conf);        //job.setJarByClass(MapJoin.class);        job.setJar("/Users/inequality/IdeaProjects/mapr/out/artifacts/mapr_jar/mapr.jar");        job.setMapperClass(MapJoinMapper.class);        job.setOutputKeyClass(Text.class);        job.setOutputKeyClass(NullWritable.class);        job.setNumReduceTasks(0);        FileInputFormat.setInputPaths(job, new Path(args[0]));        FileOutputFormat.setOutputPath(job, new Path(args[1]));        FileSystem fs = FileSystem.get(conf);        if (fs.exists(new Path(args[1]))) {            fs.delete(new Path(args[1]),true);        }        //指定一个文件到所有maptask运行节点工作目录        job.addCacheFile(new URI("file:///Users/inequality/tmp/input/join/pro.txt"));        Boolean res = job.waitForCompletion(true);        System.exit(res ? 0 : 1);    }}

阅读全文

0 0