Spring For Hadoop学习笔记（3）

绑spring跑Hadoop的案例已经跑成功，下面贴个代码。

CountMapper.java:

CountReducer.java:

<pre name="code" class="java">import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;import java.io.IOException;import java.util.StringTokenizer;/** * Created by hadoop on 15-5-21. */public class CountMapper extends Mapper<Object, Text, Text, IntWritable> {    private static final IntWritable one = new IntWritable(1);    private Text word = new Text();    public CountMapper() {    }    public void map(Object key, Text value, Context context)            throws IOException, InterruptedException {        StringTokenizer itr = new StringTokenizer(value.toString());        while (itr.hasMoreTokens()) {            this.word.set(itr.nextToken());            context.write(this.word, one);        }    }}

import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;import java.io.IOException;import java.util.Iterator;/** * Created by hadoop on 15-5-21. */public class CountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {    private IntWritable result = new IntWritable();    public CountReducer() {    }    public void reduce(Text key, Iterable<IntWritable> values,                       Context context) throws IOException, InterruptedException {        int sum = 0;        IntWritable val;        for (Iterator i$ = values.iterator(); i$.hasNext(); sum += val                .get()) {            val = (IntWritable) i$.next();        }        this.result.set(sum);        context.write(key, this.result);    }}spring-config.xml:<pre name="code" class="html"><?xml version="1.0" encoding="UTF-8"?><beans xmlns="http://www.springframework.org/schema/beans"       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"       xmlns:hdp="http://www.springframework.org/schema/hadoop"       xmlns:context="http://www.springframework.org/schema/context"       xsi:schemaLocation="        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd        http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-4.1.xsd">    <hdp:configuration>        fs.defaultFS=${hd.fs}        hadoop.tmp.dir=${java.io.tmpdir}    </hdp:configuration>    <hdp:job id="word-count"             input-path="hdfs://localhost:9000/input/"             output-path="hdfs://localhost:9000/output/"             mapper="com.jsnu.chw.hadoop.wordcount.CountMapper"             reducer="com.jsnu.chw.hadoop.wordcount.CountReducer"             combiner="com.jsnu.chw.hadoop.wordcount.CountReducer"             key="org.apache.hadoop.io.Text"             value="org.apache.hadoop.io.IntWritable"             />    <hdp:job-runner id="job-runner" job-ref="word-count" run-at-startup="true"/>    <context:property-placeholder location="./hadoop.properties"/>    <context:property-placeholder location="./log4j.properties"/></beans>

hadoop.properties:

hd.fs=hdfs://localhost:9000java.io.tmpdir=/home/hadoop/tmp

main.java:

import org.apache.hadoop.mapreduce.Job;import org.springframework.context.ApplicationContext;import org.springframework.context.support.ClassPathXmlApplicationContext;/** * Created by hadoop on 15-5-21. */public class main {    public static void main(String[] args) {        ApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:spring-config.xml");        Job job = (Job) ctx.getBean("word-count");    }}

wordcount就是hadoop自带的例子。