Hadoop2.4 MapReduce本地调试

来源：互联网发布：php配置环境工具编辑：程序博客网时间：2024/06/07 08:07

1.下载Hadoop2.4安装包

2.下载winutils等本地依赖 https://github.com/srccodes/hadoop-common-2.2.0-bin

3.将hadoop-common-2.2.0-bin-master.zip中包含winutils等文件的bin目录与Hadoop2.4 安装包中的bin合并(相同的不用覆盖)。

4.将hadoop-common-2.2.0-bin-master.zip bin目录下的hadoop.dll放入到windows/system32中去

5.安装Eclipse插件

import java.io.IOException;import java.text.SimpleDateFormat;import java.util.Date;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class LocalWordCount {private static final String INPUT_PATH="./test/in";private static final String TIME = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());private static final String OUTPUT_PATH="./test/out_"+TIME;private static final String HADOOP_HOME="D:/eclipse/hadoop-2.4.1";private static final String HADOOP_HOME_DIR="D:/eclipse/hadoop-2.4.1";public static class TokenizerMapper extendsMapper<Object, Text, Text, IntWritable> {private final static IntWritable one = new IntWritable(1);private Text word = new Text();public void map(Object key, Text value, Context context)throws IOException, InterruptedException {StringTokenizer itr = new StringTokenizer(value.toString());while (itr.hasMoreTokens()) {word.set(itr.nextToken());context.write(word, one);}}}public static class IntSumReducer extendsReducer<Text, IntWritable, Text, IntWritable> {private IntWritable result = new IntWritable();public void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {int sum = 0;for (IntWritable val : values) {sum += val.get();}result.set(sum);context.write(key, result);}}public static void main(String[] args) throws Exception {System.setProperty("HADOOP_HOME", HADOOP_HOME);System.setProperty("hadoop.home.dir", HADOOP_HOME_DIR);/** * 连接hadoop集群配置 */Configuration conf = new Configuration(true);Job job = new Job(conf, "word count");job.setJarByClass(LocalWordCount.class);// job.setNumReduceTasks(5);job.setMapperClass(TokenizerMapper.class);job.setCombinerClass(IntSumReducer.class);job.setReducerClass(IntSumReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);System.out.println("Job start!");FileInputFormat.addInputPath(job, new Path(INPUT_PATH));FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));if (job.waitForCompletion(true)) {System.out.println("ok!");} else {System.out.println("error!");System.exit(0);}}}

0 0