第一个mapreduce

hadoop环境搭好了，那么就试着写第一个mapreduce吧，以<hadoop: the definitive guide>中按年统计最高温度为例。温度文件的具体格式参考该书目。

原作者提供了温度文件的sample.txt，有5行记录作为测试，上传服务器。

Code和书中基本一致，书中的例子使用的早期的JobClient API，这里改为Job，

1. 建立项目

    在eclipse中新建一个java project，引入hadoop -core.jar.

    建3个类mapper:    MaxTemperatureMapper，reducer:MaxTemperatureReducer，主程序:MaxTemperature，然后打成jar包，上传到hadoop机器上即可。

    在hadoop上运行：

        hadoop jar ~/jars/maxTemperature.jar MaxTemperature ~/data/test/sample.txt output

 

附录：

    

MaxTemperatureMapper.java

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

public class MaxTemperatureMapper extends Mapper<LongWritable,Text,Text,IntWritable>{

    private static final int MISSING=9999;

    public void map(LongWritable key,Text value,Context context)

        throws IOException,InterruptedException{

            String line=value.toString();

            String year=line.substring(15,19);

            int temperature;

            if(line.charAt(87)=='+'){

                temperature=Integer.parseInt(line.substring(87,92));

            }else {

                temperature=Integer.parseInt(line.substring(87,92));

            }

            String quality=line.substring(92,93);

            //正则匹配

            if(temperature!=MISSING && quality.matches("[01459]")){

                context.write(new Text(year), new IntWritable(temperature));

            }

    }

}

MaxTemperatureReducer.java 

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

public class MaxTemperatureReducer extends Reducer<Text,IntWritable,Text,IntWritable>{

    public void reduce(Text key,Iterable<IntWritable> values,Context context)

        throws IOException,InterruptedException{

        int maxTemp=Integer.MIN_VALUE;

        for(IntWritable temp:values){

            maxTemp=Math.max(maxTemp, temp.get());

        }

        context.write(key, new IntWritable(maxTemp));

    }

}

 

MaxTemperature.java

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 

public class MaxTemperature {

    public static void main(String[] args) throws Exception{

        

        if(args.length!=2){

            System.err.println("usage: MaxTemperature <input path> <output path>");

            System.exit(-1);

        }        

        long begainTime=System.currentTimeMillis();       

        Job job=new Job();

        job.setJarByClass(MaxTemperature.class);        

        FileInputFormat.addInputPath(job, new Path(args[0]));

        FileOutputFormat.setOutputPath( job, new Path(args[1]));        

        job.setMapperClass(MaxTemperatureMapper.class);

        job.setReducerClass(MaxTemperatureReducer.class);

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(IntWritable.class);

        boolean status=job.waitForCompletion(true);

        

        System.out.printf("runing time(ms) : %d",System.currentTimeMillis()-begainTime);

        System.exit(status?0:1);

    }

}