使用MapReduce实现简单的倒排索引
来源:互联网 发布:stlink 怎么接单片机 编辑:程序博客网 时间:2024/06/16 04:15
package InvertedIndexer;
import java.io.IOException;
/*
* 简单的倒排索引
* Map的输出是
* (word,filename#偏移地址)
* Reduce的过程就是
* (word,fielename#偏移地址,filename#偏移地址)
* */
public class SimpleInvertedIndex {
/** 对文本进行处理,得到<word,filename#offset>格式的键值对输出,从而得到一个单词在文档中出现的位置 **/
public static class InvertedIndexMapper extends
Mapper<Object, Text, Text, Text> {
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
FileSplit fileSplit = (FileSplit) context.getInputSplit();
String fileName = fileSplit.getPath().getName(); // 得到文件名
Text word = new Text();
Text fileName_lineOffset = new Text(fileName + "#" + key.toString());
StringTokenizer itr = new StringTokenizer(value.toString());
for (; itr.hasMoreTokens();) {
word.set(itr.nextToken());
context.write(word, fileName_lineOffset);
}
}
}
/** 从Mapper处得到的内容,根据相同key值,进行累加处理,输出该单词所有出现的文档位置 **/
public static class InvertedIndexReducer extends
Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
Iterator<Text> it = values.iterator();
StringBuilder all = new StringBuilder();
if (it.hasNext())
all.append(it.next().toString());
for (; it.hasNext();) {
all.append(";");
all.append(it.next().toString());
}
context.write(key, new Text(all.toString()));
} // 最终输出键值对示例:("fish", "doc1#0; doc1#8;doc2#0;doc2#8 ")
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf, "invert index");
job.setJarByClass(SimpleInvertedIndex.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(InvertedIndexMapper.class);
job.setReducerClass(InvertedIndexReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
import java.io.IOException;
/*
* 简单的倒排索引
* Map的输出是
* (word,filename#偏移地址)
* Reduce的过程就是
* (word,fielename#偏移地址,filename#偏移地址)
* */
public class SimpleInvertedIndex {
/** 对文本进行处理,得到<word,filename#offset>格式的键值对输出,从而得到一个单词在文档中出现的位置 **/
public static class InvertedIndexMapper extends
Mapper<Object, Text, Text, Text> {
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
FileSplit fileSplit = (FileSplit) context.getInputSplit();
String fileName = fileSplit.getPath().getName(); // 得到文件名
Text word = new Text();
Text fileName_lineOffset = new Text(fileName + "#" + key.toString());
StringTokenizer itr = new StringTokenizer(value.toString());
for (; itr.hasMoreTokens();) {
word.set(itr.nextToken());
context.write(word, fileName_lineOffset);
}
}
}
/** 从Mapper处得到的内容,根据相同key值,进行累加处理,输出该单词所有出现的文档位置 **/
public static class InvertedIndexReducer extends
Reducer<Text, Text, Text, Text> {
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
Iterator<Text> it = values.iterator();
StringBuilder all = new StringBuilder();
if (it.hasNext())
all.append(it.next().toString());
for (; it.hasNext();) {
all.append(";");
all.append(it.next().toString());
}
context.write(key, new Text(all.toString()));
} // 最终输出键值对示例:("fish", "doc1#0; doc1#8;doc2#0;doc2#8 ")
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf, "invert index");
job.setJarByClass(SimpleInvertedIndex.class);
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(InvertedIndexMapper.class);
job.setReducerClass(InvertedIndexReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
0 0
- 使用MapReduce实现简单的倒排索引
- MapReduce倒排索引简单实现
- mapreduce实现搜索引擎简单的倒排索引
- 使用MapReduce实现复杂的倒排索引
- MapReduce 倒排索引的实现
- mapreduce实现倒排索引
- MapReduce实现倒排索引
- mapreduce实现倒排索引
- MapReduce倒排索引实现
- MapReduce实现倒排索引
- hadoop实现简单的倒排索引
- 倒排索引的简单实现
- MapReduce编程之倒排索引的实现
- 倒排索引的分布式实现(MapReduce程序)
- MapReduce编程之倒排索引的实现
- Hadoop2.7.3 mapreduce(四)倒排索引的实现
- MapReduce实现倒排索引全排序
- Mapreduce实现倒排索引建立
- 根据给定行号获取plaintextedit内容
- Android M 新的运行时权限开发者需要知道的一切
- Java学习笔记(45)------------Hashcode的作用
- Toast的各种样式用法
- uva 10815
- 使用MapReduce实现简单的倒排索引
- Java学习笔记(46)------------知识学习宝库!
- 【Access2003】自增列的创建
- CSS盒子模型
- JAVA安装及‘Javac’ 不是内部或外部命令
- Flex之URLLoader加载XML
- requestWindowFeature()的使用
- 欢迎使用CSDN-markdown编辑器
- SQL 第一课