MR案例之WordCount

1、在Eclipse上编写程序

1.1、导入hadoop的jar包

1.2、项目结构

WordCountMapper.java

package com.matrix.mr;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {    // 循环调用的方法，从split中每一行调用一次    @Override    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {        String[] words = value.toString().split(" ");        for (int i = 0; i < words.length; i++) {            String word = words[i];            context.write(new Text(word), new IntWritable(1));        }    }}

WordCountReduce.java

package com.matrix.mr;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {    // reduce也是循环调用，每一组调用一次    @Override    protected void reduce(Text arg0, Iterable<IntWritable> arg1, Context arg2)            throws IOException, InterruptedException {        // 根据键进行分组，        int sum = 0;        for (IntWritable i : arg1) {            sum = sum + i.get();        }        arg2.write(arg0, new IntWritable(sum));    }}

RunJob.java

package com.matrix.mr;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class RunJob {    public static void main(String[] args) {        // 读取配置文件        Configuration conf = new Configuration();        try {            FileSystem fs = FileSystem.get(conf);            Job job = Job.getInstance(conf);            // 设置用户名称            job.setJobName("wc");            // 设置任务的入口类            job.setJarByClass(RunJob.class);            job.setMapperClass(WordCountMapper.class);            job.setReducerClass(WordCountReduce.class);            job.setMapOutputKeyClass(Text.class);            job.setMapOutputValueClass(IntWritable.class);            // 设置输入数据目录            FileInputFormat.addInputPath(job, new Path("/usr/matrix/input/wc"));            // 针对目录进行判断            Path outdir = new Path("/usr/matrix/output/wc");            if (fs.exists(outdir)) {                fs.delete(outdir, true);            }            // 设置输出数据目录            // path 一个目录 而且不能存在            FileOutputFormat.setOutputPath(job, outdir);            boolean f = job.waitForCompletion(true);            if (f) {                System.out.println("WordCount程序运行成功！");            }        } catch (Exception e) {            e.printStackTrace();        }    }}

wc.java

hadoop hello worldhello hivehadoop worldhello hbase

2、导出jar包

3、将jar包上传至node1主机/opt/modules/hadoop-2.5.1目录下

4、用命令运行jar包

[root@node1 hadoop-2.5.1]# hadoop jar wc.jar com.matrix.mr.RunJob