自定义ArrayWritable

来源:互联网 发布:linux下的socket编程 编辑:程序博客网 时间:2024/05/29 19:14

*主要在 重写 readFields 时要定义values的长度,我在这卡了很久。

public static class MyMapper extends Mapper<LongWritable,Text,Text,IntArrayWritable>{@Overrideprotected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntArrayWritable>.Context context)throws IOException, InterruptedException {String[] splited = value.toString().split("\t");String msisdn = splited[1];Text k2 = new Text(msisdn);String[] array=new String[4];array[0]=splited[6];array[1]=splited[7];array[2]=splited[8];array[3]=splited[9];context.write(k2, new IntArrayWritable(array));}}public static class MyReduer extends Reducer<Text,IntArrayWritable,Text,NullWritable>{String k3=null;@Overrideprotected void reduce(Text k2, Iterable<IntArrayWritable> v2s,Reducer<Text, IntArrayWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {long upPackNum = 0L;long downPackNum = 0L;long upPayLoad = 0L;long downPayLoad = 0L;for (IntArrayWritable kpiWritable : v2s) {String[] writable= kpiWritable.toStrings();upPackNum += Integer.parseInt(writable[0]);downPackNum += Integer.parseInt(writable[1]);upPayLoad += Integer.parseInt(writable[2]);downPayLoad += Integer.parseInt(writable[3]);} k3=k2+"\t"+upPackNum+"\t"+downPackNum+"\t"+upPayLoad+"\t"+downPayLoad;       k2=new Text(k3);context.write(k2, NullWritable.get());}}public static void main(String[] args) throws Exception{Configuration conf=new Configuration();Job job=Job.getInstance(conf,TrafficStatistics2.class.getSimpleName());job.setJarByClass(TrafficStatistics2.class);job.setMapperClass(MyMapper.class);job.setReducerClass(MyReduer.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntArrayWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(NullWritable.class);FileInputFormat.setInputPaths(job,new Path("hdfs://193.124.1.120:9000/HTTP_20130313143750.dat"));FileOutputFormat.setOutputPath(job,new Path("hdfs://193.124.1.120:9000/TrafficStatistics_out1"));job.waitForCompletion(true);}}class IntArrayWritable extends ArrayWritable {private Long[] values=null;    public IntArrayWritable() {      super(LongWritable.class);     }        public IntArrayWritable(String[] strings) {    super(LongWritable.class);    values=new Long[strings.length];        for (int i = 0; i < strings.length; i++) {        values[i] =Long.parseLong(strings[i]);        }              }        @Override    public String[] toStrings() {        String[] strings = new String[values.length];        for (int i = 0; i < values.length; i++) {          strings[i] = values[i]+"";        }        return strings;      }            @Override    public void readFields(DataInput in) throws IOException {    values = new Long[4];    for (int i = 0; i < values.length; i++) {        values[i] = in.readLong();                          // store it in values      }    }    @Override    public void write(DataOutput out) throws IOException {      for (int i = 0; i < values.length; i++) {        out.writeLong(values[i]);      }    }    


0 0