partioner编程

分区编程：

1.根据业务需要，产生多个输出文件

2.多个reduce任务在运行，提高整体job的运行效率

package partition;import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.Writable;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Mapper.Context;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;/** * 分区的例子必须打成jar运行 * 用处：1.根据业务需要，产生多个输出文件 * 2.多个reduce任务在运行，提高整体job的运行效率 */public class KpiApp {private static final String INPUT_PATH = "hdfs://hadoop:9000/wlan";private static final String OUT_PATH = "hdfs://hadoop:9000/out";public static void main(String[] args) throws Exception {Job job = new Job(new Configuration(), KpiApp.class.getSimpleName());//1.1 指定输入文件路径FileInputFormat.setInputPaths(job, INPUT_PATH);//指定哪个类用来格式化输入文件job.setInputFormatClass(TextInputFormat.class);//1.2 指定自定义的Mapper类job.setMapperClass(MyMapper.class);//指定输入<k2,v2>的类型job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(KpiWritable.class);//1.3 指定分区类job.setPartitionerClass(KpiPartitioner.class);job.setNumReduceTasks(2);//1.4 排序、分区//1.5 合并//2.2 指定自定义的reduce类job.setReducerClass(MyReducer.class);//指定输出<k3,v3>的类型job.setOutputKeyClass(Text.class);job.setOutputValueClass(KpiWritable.class);//2.3 指定输出到哪里FileOutputFormat.setOutputPath(job, new Path(OUT_PATH));//设定输出文件的格式化类job.setOutputFormatClass(TextOutputFormat.class);//把代码嫁给JobTracker执行job.waitForCompletion(true);}static class MyMapper extends Mapper<LongWritable, Text, Text, KpiWritable>{@Overrideprotected void map(LongWritable key, Text value,org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, KpiWritable>.Context context)throws IOException, InterruptedException {// TODO Auto-generated method stubString[] splited = value.toString().split("\t");String msisdn = splited[1];Text k2 = new Text(msisdn);KpiWritable v2 = new KpiWritable(splited[6],splited[7],splited[8],splited[9]);context.write(k2,v2);}}static class MyReducer extends Reducer<Text, KpiWritable, Text, KpiWritable>{@Overrideprotected void reduce(Text k2, Iterable<KpiWritable> v2s,org.apache.hadoop.mapreduce.Reducer<Text, KpiWritable, Text, KpiWritable>.Context context)throws IOException, InterruptedException {// TODO Auto-generated method stublong upPackNum = 0L;long downPackNum = 0L;long upPayLoad = 0L;long downPayLoad = 0L;for (KpiWritable kpiWritable : v2s) {upPackNum += kpiWritable.upPackNum;downPackNum += kpiWritable.downPackNum;upPayLoad += kpiWritable.upPayLoad;downPayLoad += kpiWritable.downPayLoad;}KpiWritable v3 = new KpiWritable(upPackNum+"",downPackNum+"", upPayLoad+"", downPayLoad+"");context.write(k2,v3);}}static class KpiPartitioner extends HashPartitioner<Text, LongWritable>{@Overridepublic int getPartition(Text key, LongWritable value, int numReduceTasks) {// TODO Auto-generated method stubreturn (key.toString().length()==11)?0:1;}}}class  KpiWritable implements Writable{long upPackNum;long downPackNum;long upPayLoad;long downPayLoad;public KpiWritable(){}public KpiWritable(String upPackNum, String downPackNum,  String upPayLoad, String downPayLoad){this.upPackNum = Long.parseLong(upPackNum);this.downPackNum = Long.parseLong(downPackNum);this.upPayLoad = Long.parseLong(upPayLoad);this.downPayLoad = Long.parseLong(downPayLoad);}@Overridepublic void readFields(DataInput in) throws IOException {// TODO Auto-generated method stubthis.upPackNum = in.readLong();this.downPackNum = in.readLong();this.upPayLoad = in.readLong();this.downPayLoad = in.readLong();}@Overridepublic void write(DataOutput out) throws IOException {// TODO Auto-generated method stubout.writeLong(upPackNum);out.writeLong(downPackNum);out.writeLong(upPayLoad);out.writeLong(downPayLoad);}@Overridepublic String toString() {// TODO Auto-generated method stubreturn upPackNum + "\t" + downPackNum + "\t" + upPayLoad + "\t" + downPayLoad;}}