一个使用MRUnit测试mapreduce程序例子

mrunit测试mapreduce程序很方便，具体的流程和解释，给一个连接：

https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

下面把自己写的例子代码贴一下，做个笔记：

import java.io.IOException;import java.util.StringTokenizer;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.Mapper.Context;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;import java.util.regex.*;public class RetrievePasswdMapper extends Mapper<Object, Text, Text, Text>{        private String retrieveByMobile = "\\[Retrieve Password Service\\]retrieve by mobile";    private String retrieveByEmail = "\\[Retrieve Password Service\\]retrieve by email";    private String mobileNotExist = "\\[Retrieve Password Service\\]mobile not exist";    private String emailNotExist = "\\[Retrieve Password Service\\]email not exist";    private String sendMobileVcode = "\\[Retrieve Password Service\\]send mobile vcode";    private String sendEmailVcode = "\\[Retrieve Password Service\\]send email vcode";    private String successByMobile = "\\[Retrieve Password Service\\]success by mobile";    private String successByEmail = "\\[Retrieve Password Service\\]success by email";    private String failByMobile = "\\[Retrieve Password Service\\]fail by mobile";    private String failByEmail = "\\[Retrieve Password Service\\]fail by email";        Pattern p1=Pattern.compile(retrieveByMobile);    Pattern p2=Pattern.compile(retrieveByEmail);    Pattern p3=Pattern.compile(mobileNotExist);    Pattern p4=Pattern.compile(emailNotExist);    Pattern p5=Pattern.compile(sendMobileVcode);    Pattern p6=Pattern.compile(sendEmailVcode);    Pattern p7=Pattern.compile(successByMobile);    Pattern p8=Pattern.compile(successByEmail);    Pattern p9=Pattern.compile(failByMobile);    Pattern p10=Pattern.compile(failByEmail);        private Text reportDate = new Text();    private Text word = new Text();                public void map(Object key, Text value, Context context                    ) throws IOException, InterruptedException {        String line = value.toString();        reportDate.set(line.substring(1, 11).trim());        if (p1.matcher(line).find()){//            System.out.println("retrieveByMobile");//            word.set("retrieveByMobile");            context.write(reportDate, new Text("retrieveByMobile"));        } else if (p2.matcher(line).find()){            System.out.println("retrieveByEmail");            context.write(reportDate, new Text("retrieveByEmail"));        }else if (p3.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("mobileNotExist"));        }else if (p4.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("emailNotExist"));        }else if (p5.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("sendMobileVcode"));        }else if (p6.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("sendEmailVcode"));        }else if (p7.matcher(line).find()){            context.write(reportDate, new Text("successByMobile"));        }else if (p8.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("successByEmail"));        }else if (p9.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("failByMobile"));        }else if (p10.matcher(line).find()){            System.out.println("");            context.write(reportDate, new Text("failByEmail"));        }    }         /*  public static void main(String[] args)    {        String re = "[2012-08-29 00:18:39  INFO GetPasswordService:202]  [Retrieve Password Service]send mobile vcode.mobile=189XXXX0798";        String sendMobileVcode = "\\[Retrieve Password Service\\]send mobile ";        Pattern p=Pattern.compile(sendMobileVcode);        Matcher m=p.matcher(re);        System.out.println("result is "+Pattern.matches(sendMobileVcode, re));    }*/}public class RetrievePasswdReducer extends Reducer<Text,Text,Text,Text> {       public void reduce(Text key, Iterable<Text> values,                       Context context                       ) throws IOException, InterruptedException {      int[] result = new int[10];      for (Text val : values) {        if (val.toString().equals("retrieveByMobile")){            result[0]++;        }        else if (val.toString().equals("retrieveByEmail"))            result[1]++;        else if (val.toString().equals("mobileNotExist"))            result[2]++;        else if (val.toString().equals("emailNotExist"))            result[3]++;        else if (val.toString().equals("sendMobileVcode"))            result[4]++;        else if (val.toString().equals("sendEmailVcode"))            result[5]++;        else if (val.toString().equals("successByMobile"))            result[6]++;        else if (val.toString().equals("successByEmail"))            result[7]++;        else if (val.toString().equals("failByMobile"))            result[8]++;        else if (val.toString().equals("failByEmail"))            result[9]++;      }      String sum = result[0]+", "+result[1]+", "+result[2]+", "+      result[3]+", "+result[4]+", "+result[5]+", "+      result[6]+", "+result[7]+", "+result[8]+", "+result[9];      System.out.println(sum);      System.out.println("sum");      context.write(key, new Text(sum));    }  }public class RetrievePasswd {    public static void main(String[] args) {        Configuration conf = new Configuration();        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();        if (otherArgs.length != 2) {          System.err.println("Usage: wordcount <in> <out>");          System.exit(2);        }        try {            Job job = new Job(conf, "word count");            job.setJarByClass(RetrievePasswd.class);            job.setMapperClass(RetrievePasswdMapper.class);            job.setCombinerClass(RetrievePasswdReducer.class);            job.setReducerClass(RetrievePasswdReducer.class);            job.setOutputKeyClass(Text.class);            job.setOutputValueClass(Text.class);            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));            System.exit(job.waitForCompletion(true) ? 0 : 1);        } catch (Exception e)        {            e.printStackTrace();        }      }    }mrunit测试：import java.util.ArrayList;import java.util.List;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mrunit.mapreduce.MapDriver;import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;import org.junit.Before;import org.junit.Test;public class RetrievePasswdTest {    MapDriver<Object, Text, Text, Text> mapDriver;    ReduceDriver<Text, Text, Text, Text> reduceDriver;    MapReduceDriver<Object, Text, Text, Text, Text, Text> mapReduceDriver;        @Before    public void setUp() {        RetrievePasswdMapper mapper = new RetrievePasswdMapper();        RetrievePasswdReducer reducer = new RetrievePasswdReducer();        mapDriver = MapDriver.newMapDriver(mapper);        reduceDriver = ReduceDriver.newReduceDriver(reducer);        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);    }        @Test    public void testMapper() {        mapDriver.withInput(new Object(), new Text(                "[2012-08-29 00:18:39  INFO GetPasswordService:202]  [Retrieve Password Service]send mobile vcode.mobile=189XXXX0798"                ));        mapDriver.withOutput(new Text("2012-08-29"), new Text("sendMobileVcode"));        mapDriver.runTest();            }    @Test    public void testReducer() {        List<Text> values = new ArrayList<Text>();        values.add(new Text("retrieveByMobile"));        values.add(new Text("retrieveByEmail"));        values.add(new Text("retrieveByEmail"));        values.add(new Text(""));        reduceDriver.withInput(new Text("2012-08-29"), values);        reduceDriver.withOutput(new Text("2012-08-29"), new Text("1, 2, 0, 0, 0, 0, 0, 0, 0, 0"));        reduceDriver.runTest();    }}