自定义ArrayWritable
来源:互联网 发布:linux下的socket编程 编辑:程序博客网 时间:2024/05/29 19:14
*主要在 重写 readFields 时要定义values的长度,我在这卡了很久。
public static class MyMapper extends Mapper<LongWritable,Text,Text,IntArrayWritable>{@Overrideprotected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntArrayWritable>.Context context)throws IOException, InterruptedException {String[] splited = value.toString().split("\t");String msisdn = splited[1];Text k2 = new Text(msisdn);String[] array=new String[4];array[0]=splited[6];array[1]=splited[7];array[2]=splited[8];array[3]=splited[9];context.write(k2, new IntArrayWritable(array));}}public static class MyReduer extends Reducer<Text,IntArrayWritable,Text,NullWritable>{String k3=null;@Overrideprotected void reduce(Text k2, Iterable<IntArrayWritable> v2s,Reducer<Text, IntArrayWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {long upPackNum = 0L;long downPackNum = 0L;long upPayLoad = 0L;long downPayLoad = 0L;for (IntArrayWritable kpiWritable : v2s) {String[] writable= kpiWritable.toStrings();upPackNum += Integer.parseInt(writable[0]);downPackNum += Integer.parseInt(writable[1]);upPayLoad += Integer.parseInt(writable[2]);downPayLoad += Integer.parseInt(writable[3]);} k3=k2+"\t"+upPackNum+"\t"+downPackNum+"\t"+upPayLoad+"\t"+downPayLoad; k2=new Text(k3);context.write(k2, NullWritable.get());}}public static void main(String[] args) throws Exception{Configuration conf=new Configuration();Job job=Job.getInstance(conf,TrafficStatistics2.class.getSimpleName());job.setJarByClass(TrafficStatistics2.class);job.setMapperClass(MyMapper.class);job.setReducerClass(MyReduer.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntArrayWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(NullWritable.class);FileInputFormat.setInputPaths(job,new Path("hdfs://193.124.1.120:9000/HTTP_20130313143750.dat"));FileOutputFormat.setOutputPath(job,new Path("hdfs://193.124.1.120:9000/TrafficStatistics_out1"));job.waitForCompletion(true);}}class IntArrayWritable extends ArrayWritable {private Long[] values=null; public IntArrayWritable() { super(LongWritable.class); } public IntArrayWritable(String[] strings) { super(LongWritable.class); values=new Long[strings.length]; for (int i = 0; i < strings.length; i++) { values[i] =Long.parseLong(strings[i]); } } @Override public String[] toStrings() { String[] strings = new String[values.length]; for (int i = 0; i < values.length; i++) { strings[i] = values[i]+""; } return strings; } @Override public void readFields(DataInput in) throws IOException { values = new Long[4]; for (int i = 0; i < values.length; i++) { values[i] = in.readLong(); // store it in values } } @Override public void write(DataOutput out) throws IOException { for (int i = 0; i < values.length; i++) { out.writeLong(values[i]); } }
0 0
- 自定义ArrayWritable
- ArrayWritable实例
- sequencefile 和arraywritable问题解
- map/reduce 下ArrayWritable 的使用
- NoSuchMethodException: org.apache.hadoop.io.ArrayWritable.<init>问题解决总结
- 自定义
- 自定义
- 自定义
- 自定义
- 自定义
- 自定义TexBox,自定义ComboBox
- 自定义View自定义属性
- 自定义View自定义属性
- 自定义View自定义属性
- 自定义View自定义属性
- 自定义一个自定义对话框
- 自定义控件-自定义吐司
- 自定义checkbox,自定义button
- HDOJ 1203 I NEED A OFFER!(01背包)
- LeetCode题解:Count Primes
- POJ3254Corn Fields【状压dp】
- 几行和一千年
- C++强制类型转换操作符 dynamic_cast
- 自定义ArrayWritable
- GridView学习笔记
- 简单化——利益和快乐2
- 8+查找一个旋转数组的最小元素
- Java心得21
- C语言(7)--高精度加法、减法、乘法、今天是星期几、四位平方数、候选人选票问题
- hdu5418 BestCoder Round #52 (div.2) Victor and World ( floyd+状压dp)
- poj1990MooFest(树状数组)
- 应对非均衡数据集分类问题的八大策略