hadoop的MapReduce在java驱动代码

统计文件中单词出现的次数,文件text

1、java 代码，WcMapper.class继承Mapper可执行业务代码

package com.jxl.mr;import java.io.IOException;import java.util.StringTokenizer;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class WcMapper extends Mapper<LongWritable, Text, Text, IntWritable>{    //每次调用map方法会传入split中的一行数据key：改行数据所在文件中的位置下标，value 是这行数据    protected void map(LongWritable key, Text value,            Context context)            throws IOException, InterruptedException {        final String line = value.toString();        final StringTokenizer stringTokenizer = new StringTokenizer(line);        while(stringTokenizer.hasMoreTokens()){            final String word = stringTokenizer.nextToken();            context.write(new Text(word), new IntWritable(1));//map输出        }    }}

2、java 代码，WcReduce.class继承Reducer可执行业务代码

package com.jxl.mr;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;public class WcReduce extends Reducer<Text, IntWritable, Text, IntWritable>{    //重写reduce    protected void reduce(Text key, Iterable<IntWritable> iterable,            Context context)            throws IOException, InterruptedException {        int sum = 0;        for(IntWritable i:iterable){            sum = sum+i.get();        }        //输出        context.write(key, new IntWritable(sum));    }}

3、java 代码，JobRun.class执行的JobTracker

package com.jxl.mr;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class JobRun {    public static void main(String[] args) {        System.err.println("Job开始执行");        final Configuration config = new Configuration();        config.set("mapred.job.tracker", "centos-node6:9001");        try {            final Job job = new Job(config);            job.setJarByClass(JobRun.class);            job.setMapperClass(WcMapper.class);            job.setReducerClass(WcReduce.class);            job.setMapOutputKeyClass(Text.class);            job.setMapOutputValueClass(IntWritable.class);            //job.setNumReduceTasks(1);//任务执行次数            //mapreduce 输入数据所在的目录或者文件            FileInputFormat.addInputPath(job, new Path("/hello/input/wc/text"));            //mr执行之后的输出数据目录            FileOutputFormat.setOutputPath(job, new Path("/hello/output/wc/"));            //执行完退出，不可缺少            System.exit(job.waitForCompletion(true) ? 0 : 1);            System.err.println("Job执行完成");        } catch (Exception e) {            e.printStackTrace();        }    }}

4、打包wc.jar包执行一下命令

    ./hadoop jar /wc.jar com.jxl.mr.JobRun

5、浏览器访问：http://centos-node6:50030/jobtracker.jsp