hadoop 手机流统计

package com.Example;import java.io.DataInput;import java.io.DataOutput;import java.io.IOException;import org.apache.hadoop.io.Writable;public class DataWriter implements Writable {// updateprivate int upPackNum;private int upPayLoad;// downprivate int downPackNum;private int downPayLoad;public int getUpPackNum() {return upPackNum;}public void setUpPackNum(int upPackNum) {this.upPackNum = upPackNum;}public int getUpPayLoad() {return upPayLoad;}public void setUpPayLoad(int upPayLoad) {this.upPayLoad = upPayLoad;}public int getDownPackNum() {return downPackNum;}public void setDownPackNum(int downPackNum) {this.downPackNum = downPackNum;}public int getDownPayLoad() {return downPayLoad;}public void setDownPayLoad(int downPayLoad) {this.downPayLoad = downPayLoad;}public DataWriter() {}public void set(int upPackNum, int upPayLoad, int downPackNum,int downPayLoad) {this.upPackNum = upPackNum;this.upPayLoad = upPayLoad;this.downPackNum = downPackNum;this.downPayLoad = downPayLoad;}@Overridepublic void readFields(DataInput out) throws IOException {this.upPackNum = out.readInt();this.upPayLoad = out.readInt();this.downPackNum = out.readInt();this.downPayLoad = out.readInt();}@Overridepublic void write(DataOutput out) throws IOException {out.writeInt(upPackNum);out.writeInt(upPayLoad);out.writeInt(downPackNum);out.writeInt(downPayLoad);}@Overridepublic String toString() {return "Phone [upPackNum=" + upPackNum + ", upPayLoad=" + upPayLoad+ ", downPackNum=" + downPackNum + ", downPayLoad="+ downPayLoad + "]";}}package com.Example;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class PhoneValue {public static class PhoneValueMap extendsMapper<LongWritable, Text, Text, DataWriter> {private DataWriter phone = new DataWriter();private Text text = new Text();@Overrideprotected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {String valus = value.toString();String[] str = valus.split("/t");// 假如第二行的数据是手机号码String phoneNum = str[1];int upPackNum = Integer.valueOf(str[2]);int upPayLoad = Integer.valueOf(str[3]);int downPackNum = Integer.valueOf(str[4]);int downPayLoad = Integer.valueOf(str[5]);// set map ouput key / valuetext.set(phoneNum); // 设置需要按照哪个键进行统计phone.set(upPackNum, upPayLoad, downPackNum, downPayLoad);// set map ouputcontext.write(text, phone);}}public static class PhoneValueReduce extendsReducer<Text, DataWriter, Text, DataWriter> {private DataWriter dataWriter = new DataWriter();@Overrideprotected void reduce(Text arg0, Iterable<DataWriter> arg1, Context arg2) {int upPackNum = 0;int upPayLoad = 0;int downPackNum = 0;int downPayLoad = 0;for (DataWriter dataWriter : arg1) {upPackNum += dataWriter.getUpPackNum();upPayLoad += dataWriter.getUpPayLoad();downPackNum += dataWriter.getDownPackNum();downPayLoad += dataWriter.getDownPayLoad();}// set datewritabledataWriter.set(upPackNum, upPayLoad, downPackNum, downPayLoad);try {// set reduce ouputarg2.write(arg0, dataWriter);} catch (IOException e) {e.printStackTrace();} catch (InterruptedException e) {e.printStackTrace();}}}/*** * @param args* @return*/public int run(String args) {// get configConfiguration configuration = new Configuration();try {// create jobJob job = Job.getInstance(configuration);// set jobjob.setJarByClass(PhoneValue.class);// inputFileInputFormat.addInputPath(job, new Path(""));// mapjob.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(DataWriter.class);// reducejob.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);// ouputFileOutputFormat.setOutputPath(job, new Path(""));// submit jobboolean start = job.waitForCompletion(true);System.exit(start ? 0 : 1);} catch (IOException e) {e.printStackTrace();} catch (ClassNotFoundException e) {e.printStackTrace();} catch (InterruptedException e) {e.printStackTrace();}return 0;}}