eclipse下新建hadoop项目

此次介绍有关在eclipse当中进行文件操作及wordcount的运行

按照我写的顺序进行讲述

新建Map/Reduce Project项目WordCountDemo

新建类WordCountDemo，包为org.hadoop.examples

将hadoop源码中的wordcount粘贴过来

设置arguments

运行，结果出现output文件已存在的错误，在hadoop中删除output文件

hadoop fs -rmdir /input/test/output

然后再次运行出现错误如下：

Exception in thread "main" java.lang.NullPointerException........

进行如下修改

下载hadoop.dll和winutils.exe，添加到hadoop的bin目录下，版本一定要对应，我刚开始下的64位的，结果还是运行不了

还要修改环境变量，增加一个HADOOP_HOME为当前hadoop目录，在Path中添加%HADOOP_HOME%\bin

重启eclipse，运行

现在倒是不出错了，但是没反应，既不显示有什么错误，也不显示正确结果，让人很是头疼

然后想换个方式，试一下hdfs文件操作吧

写了一个有关文件操作的类operateHdfs

运行，显示当前用户权限受限，无法进行操作，想到更改用户名，将提示错误信息中的用户名改成登录linux操作hadoop的用户名，我的为spark

重启机器，打开eclipse，运行operateHdfs类，执行成功

接着在此尝试运行WordCountDemo类，运行成功

相关代码如下：

wordcountDemo

package org.hadoop.examples;import java.io.IOException;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;public class WordCountDemo {  public static class TokenizerMapper        extends Mapper<Object, Text, Text, IntWritable>{        private final static IntWritable one = new IntWritable(1);    private Text word = new Text();          public void map(Object key, Text value, Context context                    ) throws IOException, InterruptedException {      StringTokenizer itr = new StringTokenizer(value.toString());      while (itr.hasMoreTokens()) {        word.set(itr.nextToken());        context.write(word, one);      }    }  }    public static class IntSumReducer        extends Reducer<Text,IntWritable,Text,IntWritable> {    private IntWritable result = new IntWritable();    public void reduce(Text key, Iterable<IntWritable> values,                        Context context                       ) throws IOException, InterruptedException {      int sum = 0;      for (IntWritable val : values) {        sum += val.get();      }      result.set(sum);      context.write(key, result);    }  }  public static void main(String[] args) throws Exception {    Configuration conf = new Configuration();    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();    if (otherArgs.length < 2) {      System.err.println("Usage: wordcount <in> [<in>...] <out>");      System.exit(2);    }    Job job = new Job(conf, "word count");    job.setJarByClass(WordCountDemo.class);    job.setMapperClass(TokenizerMapper.class);    job.setCombinerClass(IntSumReducer.class);    job.setReducerClass(IntSumReducer.class);    job.setOutputKeyClass(Text.class);    job.setOutputValueClass(IntWritable.class);    for (int i = 0; i < otherArgs.length - 1; ++i) {      FileInputFormat.addInputPath(job, new Path(otherArgs[i]));    }    FileOutputFormat.setOutputPath(job,      new Path(otherArgs[otherArgs.length - 1]));    System.exit(job.waitForCompletion(true) ? 0 : 1);  }}

operateHdfs

public class operateHdfs {    public static void main(String args[]) throws IOException {        //测试 创建新文件        byte[] contents = "hello world\n--created by eclipse\n".getBytes();        createFile("/input/first.txt", contents);    //或 createFile("hdfs://192.168.137.56:9000/eclipse/first.txt", contents);     }    //1、创建新文件（直接生成指定路径下的first.txt，即：/eclipse/first.txt）    public static void createFile(String dst, byte[] contents) throws IOException {        Configuration conf = new Configuration();        System.out.println("-----------:"+conf);        conf.set("fs.defaultFS", "hdfs://192.168.16.60:9000");    //master        FileSystem fs = FileSystem.get(conf);        Path dstPath = new Path(dst); // 目标路径        // 打开一个输出流        FSDataOutputStream outputStream = fs.create(dstPath);        outputStream.write(contents);        outputStream.close();        fs.close();        System.out.println("文件创建成功！");    }}