win7+hadoop2.7.2+maven+idea

前言

今天想在win 7上搭一个Hadoop的开发环境，希望能够直联Hadoop集群并提交MapReduce任务，这里给出相关的关键配置。

步骤

关于idea的安装这里不再赘述，非常简单。

先下载、解压配置好bin和lib目录的hadoop文件，http://pan.baidu.com/s/1i5P8VKp

• 在win 7上配置Hadoop 到系统环境变量，不懂请自行百度。
• 建立maven项目，在pom文件中添加相关的依赖

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">    <modelVersion>4.0.0</modelVersion>    <groupId>CN.GDUT</groupId>    <artifactId>Hadoop</artifactId>    <version>1.0-SNAPSHOT</version>    <properties>        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>        <hadoop.version>2.7.2</hadoop.version>    </properties>    <dependencies>        <dependency>            <groupId>junit</groupId>            <artifactId>junit</artifactId>            <version>4.12</version>        </dependency>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-client</artifactId>            <version>${hadoop.version}</version>        </dependency>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-common</artifactId>            <version>${hadoop.version}</version>        </dependency>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-hdfs</artifactId>            <version>${hadoop.version}</version>        </dependency>    </dependencies></project>

• • 将Hadoop的相关配置文件添加到resources文件夹下
  
  
• 编写WordCount程序，分3个类，Map、Reduce、Driver
• Map

import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;import java.io.IOException;/** * Created by William on 2017/10/25 0025. */public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {    @Override    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {        //将maptask传给我们的文本内容先转换成String        String line = value.toString();        //根据空格将这一行切分成单词        String[] words = line.split(" ");        //将单词输出为<单词，1>        for(String word:words){            //将单词作为key，将次数1作为value，以便于后续的数据分发，可以根据单词分发，以便于相同单词会到相同的reduce task            context.write(new Text(word), new IntWritable(1));        }    }}

• Reduce

import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;import java.io.IOException;/** * Created by William on 2017/10/25 0025. */public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {    @Override    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {        int count=0;      /*Iterator<IntWritable> iterator = values.iterator();      while(iterator.hasNext()){         count += iterator.next().get();      }*/        for(IntWritable value:values){            count += value.get();        }        context.write(key, new IntWritable(count));    }}

• Driver

import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;/** * Created by William on 2017/10/25 0025. */public class WordcountDriver {    public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        //是否运行为本地模式，就是看这个参数值是否为local，默认就是local      conf.set("mapreduce.framework.name", "local");        //本地模式运行mr程序时，输入输出的数据可以在本地，也可以在hdfs上        //到底在哪里，就看以下两行配置你用哪行，默认就是file:///      /*conf.set("fs.defaultFS", "hdfs://mini1:9000/");*/      conf.set("fs.defaultFS", "file:///");        //运行集群模式，就是把程序提交到yarn中去运行        //要想运行为集群模式，以下3个参数要指定为集群上的值      /*conf.set("mapreduce.framework.name", "yarn");      conf.set("yarn.resourcemanager.hostname", "mini1");      conf.set("fs.defaultFS", "hdfs://mini1:9000/");*/        Job job = Job.getInstance(conf);//        job.setJar("c:/wc.jar");        //指定本程序的jar包所在的本地路径      job.setJarByClass(WordcountDriver.class);        //指定本业务job要使用的mapper/Reducer业务类        job.setMapperClass(WordcountMapper.class);        job.setReducerClass(WordcountReducer.class);        //指定mapper输出数据的kv类型        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(IntWritable.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(IntWritable.class);        job.setCombinerClass(WordcountReducer.class);           //指定job的输入原始文件所在目录        FileInputFormat.setInputPaths(job, new Path(args[0]));        //指定job的输出结果所在目录        FileOutputFormat.setOutputPath(job, new Path(args[1]));        //将job中配置的相关参数，以及job所用的java类所在的jar包，提交给yarn去运行        boolean res = job.waitForCompletion(true);        System.exit(res?0:1);    }}

• 设置idea运行参数

编辑好输入文本，本地集群都可以，Configuration如下图：

完成!

如果还报IO.NATIVE错误，将bin/hadoop.dll丢到system32去