通过MapReduce统计每个单子在每个文件中出现的次数（FileSplit的使用），单词作为key，所在文本和次数作为value进行统计

代码如下：

package cn.toto.bigdata.mr.index;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.FileSplit;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class IndexCreateStepOne {public static class IndexCreateMapper extends Mapper<LongWritable, Text, Text, IntWritable> {Text k = new Text();IntWritable v = new IntWritable(1);@Overrideprotected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line = value.toString();String[] words = line.split(" ");FileSplit inputSplit = (FileSplit) context.getInputSplit();//获取到word(单词)所在的文件的名称String fileName = inputSplit.getPath().getName();//最终输出的格式效果如：       key:单词---文件名   value:1for(String word : words) {k.set(word + "--" + fileName);context.write(k, v);}}}public static class IndexCreateReducer extends Reducer<Text, IntWritable, Text, IntWritable> {IntWritable v = new IntWritable();@Overrideprotected void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {int count = 0;for (IntWritable value : values) {count += value.get();}v.set(count);context.write(key, v);}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = Job.getInstance(conf);//告诉框架，我们的程序所在jar包的路径// job.setJar("c:/wordcount.jar");job.setJarByClass(IndexCreateStepOne.class);//告诉框架，我们的程序所用的mapper类和reducer类job.setMapperClass(IndexCreateMapper.class);job.setReducerClass(IndexCreateReducer.class);job.setCombinerClass(IndexCreateReducer.class);//告诉框架，我们的mapperreducer输出的数据类型job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);FileInputFormat.setInputPaths(job, new Path("E:/wordcount/inverindexinput/"));//告诉框架，我们的处理结果要输出到哪里FileOutputFormat.setOutputPath(job, new Path("E:/wordcount/index-1/"));boolean res = job.waitForCompletion(true);System.exit(res ? 0 : 1);}}

准备条件

1、要处理的数据文件

b.txt的内容如下：

其它的c.txt,d.txt和上面的类似

运行后的结果如下：

这样，可以列出各各单词在每个文件中的数量了

接着，做如下的功能：单词作为key,在文件和文件中的个数的数值作为value，然后去做统计，实例代码如下：

package cn.toto.bigdata.mr.index;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.mockito.internal.stubbing.StubbedInvocationMatcher;import io.netty.handler.codec.http.HttpHeaders.Values;public class IndexCreateStepTwo {public static class IndexCreateStepTwoMapper extends Mapper<LongWritable, Text, Text, Text> {Text k = new Text();Text v = new Text();@Overrideprotected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)throws IOException, InterruptedException {String line = value.toString();String[] fields = line.split("\t");String word_file = fields[0];String count = fields[1];String[] split = word_file.split("--");String word = split[0];String file = split[1];k.set(word);v.set(file + "--" + count);context.write(k, v);}}public static class IndexCreateStepTwoReducer extends Reducer<Text, Text, Text, Text> {Text v = new Text();@Overrideprotected void reduce(Text key, Iterable<Text> values, Context context)throws IOException, InterruptedException {StringBuffer sb = new StringBuffer();for (Text value : values) {sb.append(value.toString()).append(" ");}v.set(sb.toString());context.write(key, v);}}public static void main(String[] args) throws Exception {Configuration conf = new Configuration();Job job = Job.getInstance(conf);//告诉框架，我们的程序所在jar包的路径// job.setJar("c:/wordcount.jar");job.setJarByClass(IndexCreateStepTwo.class);//告诉框架，我们的程序所用的mapper类和reducer类job.setMapperClass(IndexCreateStepTwoMapper.class);job.setReducerClass(IndexCreateStepTwoReducer.class);job.setCombinerClass(IndexCreateStepTwoReducer.class);//告诉框架，我们的mapperreducer输出的数据类型job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(Text.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(Text.class);FileInputFormat.setInputPaths(job, new Path("E:/wordcount/index-1/"));//告诉框架，我们的处理结果要输出到哪里去FileOutputFormat.setOutputPath(job, new Path("E:/wordcount/index-2/"));boolean res = job.waitForCompletion(true);System.exit(res ? 0 : 1);}}

程序运行的结果如下：