mapreduce(二):wordcount详解

package mapreduce;import java.io.FileInputStream;import java.io.IOException;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;public class MyWordCountJob {    /**     * @author Edison Chou     * @version 1.0     * @param KEYIN     *            →k1 表示每一行的起始位置（偏移量offset）     * @param VALUEIN     *            →v1 表示每一行的文本内容     * @param KEYOUT     *            →k2 表示每一行中的每个单词     * @param VALUEOUT     *            →v2 表示每一行中的每个单词的出现次数，固定值为1     */    public static class MyMapper extends            Mapper<LongWritable, Text, Text, LongWritable> {        protected void map(LongWritable key, Text value,                Mapper<LongWritable, Text, Text, LongWritable>.Context context)                throws java.io.IOException, InterruptedException {            String[] spilted = value.toString().split(" ");            for (String word : spilted) {                context.write(new Text(word), new LongWritable(1L));            }        };    }    /**     * @author Edison Chou     * @version 1.0     * @param KEYIN     *            →k2 表示每一行中的每个单词     * @param VALUEIN     *            →v2 表示每一行中的每个单词的出现次数，固定值为1     * @param KEYOUT     *            →k3 表示每一行中的每个单词     * @param VALUEOUT     *            →v3 表示每一行中的每个单词的出现次数之和     */    public static class MyReducer extends            Reducer<Text, LongWritable, Text, LongWritable> {        protected void reduce(Text key,                java.lang.Iterable<LongWritable> values,                Reducer<Text, LongWritable, Text, LongWritable>.Context context)                throws java.io.IOException, InterruptedException {            long count = 0L;            for (LongWritable value : values) {                count += value.get();            }            context.write(key, new LongWritable(count));        };    }    // 输入文件路径    public static final String INPUT_PATH = "hdfs://hadoop-master:9000/testdir/input/words.txt";    // 输出文件路径    public static final String OUTPUT_PATH = "hdfs://hadoop-master:9000/testdir/output/wordcount";    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        // 0.0:首先删除输出路径的已有生成文件        FileSystem fs = FileSystem.get(new URI(INPUT_PATH), conf);        Path outPath = new Path(OUTPUT_PATH);        if (fs.exists(outPath)) {            fs.delete(outPath, true);        }        Job job = new Job(conf, "WordCount");        job.setJarByClass(MyWordCountJob.class);        // 1.0:指定输入目录        FileInputFormat.setInputPaths(job, new Path(INPUT_PATH));        // 1.1:指定对输入数据进行格式化处理的类（可以省略）        job.setInputFormatClass(TextInputFormat.class);        // 1.2:指定自定义的Mapper类        job.setMapperClass(MyMapper.class);        // 1.3:指定map输出的<K,V>类型（如果<k3,v3>的类型与<k2,v2>的类型一致则可以省略）        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(LongWritable.class);        // 1.4:分区（可以省略）        job.setPartitionerClass(HashPartitioner.class);        // 1.5:设置要运行的Reducer的数量（可以省略）        job.setNumReduceTasks(1);        // 1.6:指定自定义的Reducer类        job.setReducerClass(MyReducer.class);        // 1.7:指定reduce输出的<K,V>类型        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(LongWritable.class);        // 1.8:指定输出目录        FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));        // 1.9:指定对输出数据进行格式化处理的类（可以省略）        job.setOutputFormatClass(TextOutputFormat.class);        // 2.0:提交作业        boolean success = job.waitForCompletion(true);        if (success) {            System.out.println("Success");            System.exit(0);        } else {            System.out.println("Failed");            System.exit(1);        }    }}