老api写法的wordcount程序

来源:互联网 发布:淘宝快递单打印软件 编辑:程序博客网 时间:2024/06/05 03:36
package com.ccse.hadoop.old;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import java.util.Iterator;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapred.FileOutputFormat;import org.apache.hadoop.mapred.JobClient;import org.apache.hadoop.mapred.JobConf;import org.apache.hadoop.mapred.MapReduceBase;import org.apache.hadoop.mapred.Mapper;import org.apache.hadoop.mapred.OutputCollector;import org.apache.hadoop.mapred.Reducer;import org.apache.hadoop.mapred.Reporter;import org.apache.hadoop.mapred.TextInputFormat;import org.apache.hadoop.mapred.TextOutputFormat;import org.apache.hadoop.mapred.FileInputFormat;/** * 老api的写法 * @author woshiccna * */public class WordCountApp {public final static String INPUT_PATH = "hdfs://chaoren1:9000/mapinput";public final static String OUTPUT_PATH = "hdfs://chaoren1:9000/mapoutput";public static void main(String[] args) throws IOException, URISyntaxException {JobConf conf = new JobConf(WordCountApp.class);conf.setJobName("wordcount");Configuration config = new Configuration();FileSystem fileSystem = FileSystem.get(new URI(OUTPUT_PATH), config);fileSystem.delete(new Path(OUTPUT_PATH), true);conf.setMapperClass(MyMapper.class);conf.setOutputKeyClass(Text.class);conf.setOutputValueClass(LongWritable.class);conf.setInputFormat(TextInputFormat.class);FileInputFormat.setInputPaths(conf, new Path(INPUT_PATH));conf.setReducerClass(MyReducer.class);conf.setOutputKeyClass(Text.class);conf.setOutputValueClass(LongWritable.class);conf.setOutputFormat(TextOutputFormat.class);FileOutputFormat.setOutputPath(conf, new Path(OUTPUT_PATH));JobClient.runJob(conf);}public static class MyMapper extends MapReduceBase          implements Mapper<LongWritable, Text, Text, LongWritable> {private Text word = new Text();private LongWritable writable = new LongWritable(1);@Overridepublic void map(LongWritable key, Text value,OutputCollector<Text, LongWritable> output, Reporter reporter)throws IOException {if (value != null) {String line = value.toString();StringTokenizer tokenizer = new StringTokenizer(line);while (tokenizer.hasMoreElements()) {word = new Text(tokenizer.nextToken());    output.collect(word, writable);}}}}public static class MyReducer extends MapReduceBase implements         Reducer<Text, LongWritable, Text, LongWritable> {@Overridepublic void reduce(Text key, Iterator<LongWritable> values,OutputCollector<Text, LongWritable> output, Reporter reporter)throws IOException {long sum = 0;while (values.hasNext()) {LongWritable value = values.next();sum += value.get();}output.collect(key, new LongWritable(sum));}}}

0 0
原创粉丝点击