Hadoop 类Grep源代码注释

/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements.  See the NOTICE file * distributed with this work for additional information * regarding copyright ownership.  The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package com.tdxx.hadoop.example;import java.util.Random;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.*;import org.apache.hadoop.mapred.lib.*;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;/** * 从input中提取与表达式相符的单词并计算词频 *  * 继承自配置基类Configured,并扩展接口Tool * Configured类中有一个变量conf用于存储配置文件 * Tool中只有一个方法需要实现 * int run(String [] args)用于运行输入参数 * hadoop jar Grep.jar /user/hadoop/20130704/grep.txt /user/hadoop/output/ 'aaa.*' * hadoop jar Grep.jar /user/hadoop/20130704/grep.txt /user/hadoop/output/ '[a-z.]+' */public class Grep extends Configured implements Tool {// singletonprivate Grep() {} public int run(String[] args) throws Exception {if (args.length < 3) {System.out.println("Grep <inDir> <outDir> <regex> [<group>]");ToolRunner.printGenericCommandUsage(System.out);return -1;}Path tempDir = new Path("grep-temp-"+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));//创建jobJobConf grepJob = new JobConf(getConf(), Grep.class);try {//job命名grepJob.setJobName("grep-search");//设置job的输入路径FileInputFormat.setInputPaths(grepJob, args[0]);//设置Mapper类grepJob.setMapperClass(RegexMapper.class);grepJob.set("mapred.mapper.regex", args[2]);if (args.length == 4)grepJob.set("mapred.mapper.regex.group", args[3]);//设置Combiner类grepJob.setCombinerClass(LongSumReducer.class);//设置Reducer类grepJob.setReducerClass(LongSumReducer.class);//设置输出路径FileOutputFormat.setOutputPath(grepJob, tempDir);//设置输出格式grepJob.setOutputFormat(SequenceFileOutputFormat.class);//设置输出键的类grepJob.setOutputKeyClass(Text.class);//设置输出值的类grepJob.setOutputValueClass(LongWritable.class);//运行JobClient.runJob(grepJob);JobConf sortJob = new JobConf(getConf(), Grep.class);sortJob.setJobName("grep-sort");FileInputFormat.setInputPaths(sortJob, tempDir);sortJob.setInputFormat(SequenceFileInputFormat.class);sortJob.setMapperClass(InverseMapper.class);sortJob.setNumReduceTasks(1); // write a single fileFileOutputFormat.setOutputPath(sortJob, new Path(args[1]));sortJob.setOutputKeyComparatorClass // sort by decreasing freq(LongWritable.DecreasingComparator.class);JobClient.runJob(sortJob);} finally {FileSystem.get(grepJob).delete(tempDir, true);}return 0;}public static void main(String[] args) throws Exception {int res = ToolRunner.run(new Configuration(), new Grep(), args);System.exit(res);}}