MapReduce八股文模板

来源：互联网发布：tensorflow 物体识别编辑：程序博客网时间：2024/06/12 23:34
package com.bruce.mapreduce;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;public class moduleMapReduce extends Configured implements Tool {// step 1: Map Class/** * Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> *  *///TODO update paragrampublic static class ModuleMapper extendsMapper<LongWritable, Text, Text, IntWritable> {@Overrideprotected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {// TODO Auto-generated method stub}}// step 2: Reduce Class/** * Reducer<KEYIN, VALUEIN, KEYOUT, VALUEOUT> *  *///TODOpublic static class ModuleReducer extendsReducer<Text, IntWritable, Text, IntWritable> {@Overrideprotected void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {// TODO Auto-generated method stub}}// step 3: Driver ,component job, implements Toolpublic int run(String[] args) throws Exception {// 1: get configrationConfiguration configuration = getConf();// 2: create JobJob job = Job.getInstance(configuration, this.getClass().getSimpleName());// run jarjob.setJarByClass(this.getClass());// 3: set job// input -> map -> reduce -> output// 3.1 inputPath inPath = new Path(args[0]);FileInputFormat.addInputPath(job, inPath);// 3.2: mapjob.setMapperClass(ModuleMapper.class);//TODO update paragramjob.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);// 3.3: reducejob.setReducerClass(ModuleReducer.class);//TODOjob.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);// 3.4: outputPath outPath = new Path(args[1]);FileOutputFormat.setOutputPath(job, outPath);// 4: submit jobboolean isSuccess = job.waitForCompletion(true);return isSuccess ? 0 : 1;}// step 4: run programpublic static void main(String[] args) throws Exception {// 1: get configrationConfiguration configuration = new Configuration();//int status = new moduleMapReduce().run(args);int status = ToolRunner.run(configuration, new moduleMapReduce(), args);System.exit(status);}}
阅读全文
0 0
tensorflow 物体识别
原创粉丝点击