倒排索引实现

简介：参考文章：

输入输出：

输入：

a.txt:

hadoop hello world

hello test

test hadoop

b.txt:

test world

hello world test

hadoop

输出：

hadoop hdfs://h2/apps/ca/yanh/data/invertedIndex/a.txt-->2       hdfs://h2/apps/ca/yanh/data/invertedIndex/b.txt-->1 
hello hdfs://h2/apps/ca/yanh/data/invertedIndex/b.txt-->1           hdfs://h2/apps/ca/yanh/data/invertedIndex/a.txt-->2 
test hdfs://h2/apps/ca/yanh/data/invertedIndex/a.txt-->2             hdfs://h2/apps/ca/yanh/data/invertedIndex/b.txt-->2 
world hdfs://h2/apps/ca/yanh/data/invertedIndex/b.txt-->2          hdfs://h2/apps/ca/yanh/data/invertedIndex/a.txt-->1

实现代码：

<span style="font-size:12px;">import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.FileSplit;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import java.io.IOException;/** * Created by 浩 on 2015/4/20. */public class InvertedIndex {    public static void main(String[] args)throws IOException,InterruptedException, ClassNotFoundException {        Configuration conf = new Configuration();        Job job = Job.getInstance(conf);        job.setJarByClass(InvertedIndex.class);        job.setMapperClass(InvertedMapper.class);        FileInputFormat.addInputPath(job, new Path(args[0]));        job.setCombinerClass(InvertedCombiner.class);        job.setReducerClass(InvertedReducer.class);        FileOutputFormat.setOutputPath(job, new Path(args[1]));        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(Text.class);        System.exit(job.waitForCompletion(true)? 0: 1);    }}class InvertedMapper extends Mapper<LongWritable, Text, Text, Text> {    private Text k = new Text();    private Text v = new Text();    @Override    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {        String line = value.toString();        String[] words = line.split(" ");        FileSplit inputSplit = (FileSplit) context.getInputSplit();        Path path = inputSplit.getPath();        for (String word: words) {            k.set(word + "-->" + path.toString());            v.set("1");            context.write(k, v);        }    }}class InvertedCombiner extends Reducer<Text, Text, Text, Text> {    private Text k = new Text();    private Text v = new Text();    @Override    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {        String[] fields = key.toString().split("-->");        int sum = 0;        for(Text val: values) {            sum += Integer.parseInt(val.toString());        }        k.set(fields[0]);        v.set(fields[1] + "-->" + sum);        context.write(k, v);    }}class InvertedReducer extends Reducer<Text, Text, Text, Text> {    private Text k = new Text();    private Text v = new Text();    @Override    protected void reduce(Text key, Iterable<Text> values, Context context)throws IOException, InterruptedException {        String str = "";        for(Text val: values) {            str += val.toString() + "\t";        }        k.set(key);        v.set(str);        context.write(k, v);    }}</span>