Hadoop程序基础模板
来源:互联网 发布:女生学java还是web前端 编辑:程序博客网 时间:2024/06/03 21:11
输入文件格式为:X,Y,表示X引用Y;
本程序目的为将引用数据进行倒排,对于每一个专利找到引用他的那些专利
import java.io.IOException;import java.util.Iterator;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.JobConf;import org.apache.hadoop.mapred.KeyValueTextInputFormat;import org.apache.hadoop.mapred.MapReduceBase;import org.apache.hadoop.mapred.Mapper;import org.apache.hadoop.mapred.OutputCollector;import org.apache.hadoop.mapred.Reducer;import org.apache.hadoop.mapred.Reporter;import org.apache.hadoop.mapred.TextOutputFormat;import org.apache.hadoop.mapred.FileInputFormat;import org.apache.hadoop.mapred.FileOutputFormat;import org.apache.hadoop.mapred.JobClient;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;import com.google.inject.Key;import com.sun.tracing.dtrace.ArgsAttributes;public class MyJob extends Configured implements Tool { public static class MapClass extends MapReduceBase implements Mapper<Text, Text, Text, Text>{ public void map(Text key,Text value,OutputCollector<Text, Text> output,Reporter reporter)throws IOException{ output.collect(value, key); } } public static class Reduce extends MapReduceBase implements Reducer<Text, Text, Text, Text>{ public void reduce(Text key,Iterator<Text> value,OutputCollector<Text, Text> output,Reporter reporter)throws IOException{// int sum=0;// while(value.hasNext()){// value.next();// sum++;// }// output.collect(key, new IntWritable(sum)); String res=""; while(value.hasNext()){ if(res.length()>0)res+=","; res+=value.next().toString(); } output.collect(key, new Text(res)); } } @Override public int run(String[] arg0) throws Exception { // TODO Auto-generated method stub Configuration configuration=getConf(); JobConf job=new JobConf(configuration,MyJob.class); FileInputFormat.setInputPaths(job, new Path(arg0[0])); FileOutputFormat.setOutputPath(job, new Path(arg0[1])); job.setJobName("InvertedIndex"); job.setMapperClass(MapClass.class); job.setReducerClass(Reduce.class); job.setInputFormat(KeyValueTextInputFormat.class); job.setOutputFormat(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); //job.setMapOutputValueClass(Text.class); job.set("key.value.separator.in.input.line",","); JobClient.runJob(job); return 0; } public static void main(String[] args) throws Exception{ // TODO Auto-generated method stub //ToolRunner.run(conf, tool, args) int res=ToolRunner.run(new Configuration(), new MyJob(), args); System.exit(res); }}
0 0
- Hadoop程序基础模板
- Hadoop构建MapRedcue程序的基础模板
- Hadoop网络课程模板程序
- Hadoop网络课程模板程序
- 基础程序模板
- hadoop mapReduce程序模板-倒排索引
- Hadoop MapReduce程序的模板框架
- hadoop教程---WordCount基础程序分析
- Hadoop inAction] 第4章 编写MapReduce基础程序
- hadoop基础
- hadoop基础
- hadoop基础
- Hadoop基础
- Hadoop基础
- Hadoop基础
- hadoop基础
- hadoop基础
- Hadoop基础
- 新手笔记:使用final关键字修饰
- 傻瓜方法求集合的所有子集问题(java版)
- 猜数字
- 美团的估值为何下降了?
- JavaScript中的匿名函数及函数的闭包
- Hadoop程序基础模板
- Lua笔记15 __index
- Android之RecyclerView的动画详解
- This application is modifying the autolayout engine from a background thread 原因
- 让服务器iis支持.apk文件下载的设置方法
- CRF专业领域实体识别
- spark streaming接kafka数据方式汇总
- Error:Execution failed for task ':app:dexDebug'. > com.android.ide.common.process.ProcessException
- 第四周项目1:求最大公约数