Hadoop不用eclipse去编译运行WordCount

1，写程序MyWordCount.java

hduser@ubuntu:/usr/local/hadoop$ gedit MyWordCount.java

package org.myorg;import java.io.IOException;import java.util.*;import org.apache.hadoop.fs.Path;import org.apache.hadoop.conf.*;import org.apache.hadoop.io.*;import org.apache.hadoop.mapred.*;import org.apache.hadoop.util.*;public class WordCount {public static void main(String[] args) throws Exception {JobConf conf = new JobConf(WordCount.class);conf.setJobName("wordcount");//conf.setNumReduceTasks(0);conf.setOutputKeyClass(Text.class);conf.setOutputValueClass(IntWritable.class);conf.setMapperClass(Map.class);conf.setCombinerClass(Reduce.class);//conf.setReducerClass(Reduce.class);conf.setInputFormat(TextInputFormat.class);conf.setOutputFormat(TextOutputFormat.class);FileInputFormat.setInputPaths(conf, new Path(args[0]));FileOutputFormat.setOutputPath(conf, new Path(args[1]));JobClient.runJob(conf);}public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {private final static IntWritable one = new IntWritable(1);private Text word = new Text();public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {String line = value.toString();StringTokenizer tokenizer = new StringTokenizer(line);while (tokenizer.hasMoreTokens()) {word.set(tokenizer.nextToken());output.collect(word, one);}}}public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {int sum = 0;while (values.hasNext()) {sum += values.next().get();}output.collect(key, new IntWritable(sum));}}}

 

 

源文档 <http://hadoop.apache.org/docs/r0.18.3/mapred_tutorial.html>

注：conf.setInputFormat(TextInputFormat.class);    //TextInputFormat是默认的InputFormat。这说明Map类的键是LongWritable类型，存储整个文件的字节偏移量，值是Text类型，是一行内容。StringTokenizer 把行按空格拆分成单词。
conf.setOutputFormat(TextOutputFormat.class);     //输出格式为TextOutputFormat，把输出记录写成文本行。键值可以使任何类型，因为可以用toString()方法转成字符串。这里的输出键是Text类型，值是IntWritable类型。

2，编译
Mkdir  wordcountsource
hduser@ubuntu:/usr/local/hadoop$ javac -classpath hadoop-core-1.1.1.jar -d wordcountsource MyWordCount.java
编译java到wordcountsource文件夹下
如果出现错误：error while writing Map: could not create parent directories
说明没有写入input文件的权限

3，生成jar
hduser@ubuntu:/usr/local/hadoop$ sudo jar -cvf MyWordCount.jar  -C wordcountsource/ .
在当前目录下生成MyWordCount.jar

Ant打包

4，在input文件夹下创建file0和file1
hduser@ubuntu:/usr/local/hadoop/input$ mkdir input    //input文件夹为输入
hduser@ubuntu:/usr/local/hadoop/input$ sudo gedit file0
hduser@ubuntu:/usr/local/hadoop/input$ sudo gedit file1

5，运行MyWordCount.jar
hduser@ubuntu:/usr/local/hadoop$ sudo bin/hadoop jar MyWordCount.jar org.myorg.MyWordCount input output
确保有创建output的权限

6，查看结果
hduser@ubuntu:/usr/local/hadoop$ cat output/part-00000

Bye 1
Hello 1
World 2