电话号码上传下载流量的hadoop代码简单实现
来源:互联网 发布:3000 左右手机 知乎 编辑:程序博客网 时间:2024/06/06 00:19
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
public class DataCount {
public static class DCMapper extends Mapper<LongWritable, Text, Text, DataBean>{
@Override
protected void map(LongWritable key, Text value,Context context){
//key 为一个long型的数字。value为一个字符串,context为上下文的内容
String line = value.toString();
//先把hadoop里面的字符串转换成可以Java识别的字符串
String fileds[] = line.split("/t")
//按照空格分割里面的字符串
String tel = fileds[1]
//拿出里面的电话号码那一行
long up = long.parseLong(fileds[8]);
//电话上网上传流量数据的取出,需要解析成long型的数据
long down = long.parseLong(fileds[9]);
//电话上网下载流量数据的取出。也是需要解析成long型的数据
DataBean bean = new DataBean(tel,up,down)
//新建一个对象。用构造方法来存储需要的显示的信息
电话号码 : 上传流量 :下载流量
context.write(new Text(tel),bean);
//把电话号码作为key 把bean作为value输出
}
//以上为mapper方法
}
public static class DCReducer extends Reducer<Text,DataBean,Text,DataBean>{
@Override
protected void reduce (Text key,Iterable<DataBean>values,Context context){
long up_sum = 0;
long down_sum = 0;
//定义2个计数器
for(DataBean bean :values){
up_sum += bean.getUpPayLoad();
down_sum += bean.getDownPayLoad()
//累加上传和下行流量
}
DataBean bean = new DataBean("",up_sum,down_sum);
context.write(key,bean);
//把电话号码 和bean输出
}
}
public static void mian(String[] args){
Configuration conf = new Configuration()
//新建一个配置文件
Job job = Job.getInstance(conf);
//hadoop里面的自带的函数必须用Job
job.setJar(DataCount.class);
//这句话必须加进去
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(DataBean.class);
job.setMapperClass(DCMapper.class);
FileInputFormat,setInputPaths(job,new Path(""));
job.setReducerClass(DCReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DataBean.class);
FileInputFormat.setOutputPaths(job,new Path(""));
job.wait(true);
}
//下面的是写一个DataBean
//定义一些属性和方法
public class DataInfo implements Writable{
private String tel;
private long upPayLoad;
private long downPayLoad;
private long totalPayLoad;
public DataInfo(){}
public DataInfo(String tel, long upPayLoad, long downPayLoad) {
this.tel = tel;
this.upPayLoad = upPayLoad;
this.downPayLoad = downPayLoad;
this.totalPayLoad = upPayLoad + downPayLoad;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(tel);
out.writeLong(upPayLoad);
out.writeLong(downPayLoad);
out.writeLong(totalPayLoad);
}
@Override
public void readFields(DataInput in) throws IOException {
this.tel = in.readUTF();
this.upPayLoad = in.readLong();
this.downPayLoad = in.readLong();
this.totalPayLoad = in.readLong();
}
@Override
public String toString() {
return upPayLoad + "\t" + downPayLoad + "\t" + totalPayLoad;
}
public String getTel() {
return tel;
}
public void setTel(String tel) {
this.tel = tel;
}
public long getUpPayLoad() {
return upPayLoad;
}
public void setUpPayLoad(long upPayLoad) {
this.upPayLoad = upPayLoad;
}
public long getDownPayLoad() {
return downPayLoad;
}
public void setDownPayLoad(long downPayLoad) {
this.downPayLoad = downPayLoad;
}
public long getTotalPayLoad() {
return totalPayLoad;
}
public void setTotalPayLoad(long totalPayLoad) {
this.totalPayLoad = totalPayLoad;
}
}
}
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
public class DataCount {
public static class DCMapper extends Mapper<LongWritable, Text, Text, DataBean>{
@Override
protected void map(LongWritable key, Text value,Context context){
//key 为一个long型的数字。value为一个字符串,context为上下文的内容
String line = value.toString();
//先把hadoop里面的字符串转换成可以Java识别的字符串
String fileds[] = line.split("/t")
//按照空格分割里面的字符串
String tel = fileds[1]
//拿出里面的电话号码那一行
long up = long.parseLong(fileds[8]);
//电话上网上传流量数据的取出,需要解析成long型的数据
long down = long.parseLong(fileds[9]);
//电话上网下载流量数据的取出。也是需要解析成long型的数据
DataBean bean = new DataBean(tel,up,down)
//新建一个对象。用构造方法来存储需要的显示的信息
电话号码 : 上传流量 :下载流量
context.write(new Text(tel),bean);
//把电话号码作为key 把bean作为value输出
}
//以上为mapper方法
}
public static class DCReducer extends Reducer<Text,DataBean,Text,DataBean>{
@Override
protected void reduce (Text key,Iterable<DataBean>values,Context context){
long up_sum = 0;
long down_sum = 0;
//定义2个计数器
for(DataBean bean :values){
up_sum += bean.getUpPayLoad();
down_sum += bean.getDownPayLoad()
//累加上传和下行流量
}
DataBean bean = new DataBean("",up_sum,down_sum);
context.write(key,bean);
//把电话号码 和bean输出
}
}
public static void mian(String[] args){
Configuration conf = new Configuration()
//新建一个配置文件
Job job = Job.getInstance(conf);
//hadoop里面的自带的函数必须用Job
job.setJar(DataCount.class);
//这句话必须加进去
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(DataBean.class);
job.setMapperClass(DCMapper.class);
FileInputFormat,setInputPaths(job,new Path(""));
job.setReducerClass(DCReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DataBean.class);
FileInputFormat.setOutputPaths(job,new Path(""));
job.wait(true);
}
//下面的是写一个DataBean
//定义一些属性和方法
public class DataInfo implements Writable{
private String tel;
private long upPayLoad;
private long downPayLoad;
private long totalPayLoad;
public DataInfo(){}
public DataInfo(String tel, long upPayLoad, long downPayLoad) {
this.tel = tel;
this.upPayLoad = upPayLoad;
this.downPayLoad = downPayLoad;
this.totalPayLoad = upPayLoad + downPayLoad;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(tel);
out.writeLong(upPayLoad);
out.writeLong(downPayLoad);
out.writeLong(totalPayLoad);
}
@Override
public void readFields(DataInput in) throws IOException {
this.tel = in.readUTF();
this.upPayLoad = in.readLong();
this.downPayLoad = in.readLong();
this.totalPayLoad = in.readLong();
}
@Override
public String toString() {
return upPayLoad + "\t" + downPayLoad + "\t" + totalPayLoad;
}
public String getTel() {
return tel;
}
public void setTel(String tel) {
this.tel = tel;
}
public long getUpPayLoad() {
return upPayLoad;
}
public void setUpPayLoad(long upPayLoad) {
this.upPayLoad = upPayLoad;
}
public long getDownPayLoad() {
return downPayLoad;
}
public void setDownPayLoad(long downPayLoad) {
this.downPayLoad = downPayLoad;
}
public long getTotalPayLoad() {
return totalPayLoad;
}
public void setTotalPayLoad(long totalPayLoad) {
this.totalPayLoad = totalPayLoad;
}
}
}
0 0
- 电话号码上传下载流量的hadoop代码简单实现
- socket实现简单的上传下载
- hadoop的FileSystem 文件系统实现上传下载文件
- hadoop的FileSystem 文件系统实现上传下载文件
- Java实现的简单电话号码储存
- 一个简单的FTP客户端 实现上传下载
- JavaWeb实现简单的文件上传下载
- mapreduce(JAVA)实现(大数据)电话号码对应的流量排序(倒序)
- 简单的电话号码处理
- struts实现上传下载代码
- java 实现hadoop的hdfs文件的上传下载删除创建
- mapreduce实现简单的流量统计功能
- Java实现最电话号码的简单加密源码
- 简单的上传下载Demo
- 简单的哈希表存储电话号码
- MFC简单实现FTP上传下载
- 文件上传下载PHP代码实现
- 用MVC实现简单的文件(图片)上传下载功能
- ODS和数据仓库的区别
- ANDROID中ACTIVITY运行时屏幕方向与显示方式详解
- 【java】表达式树
- HDFS NameNode内存全景
- JAVA sychronized 实战操作解读
- 电话号码上传下载流量的hadoop代码简单实现
- 分类问题中的数据不平衡问题
- 麻将项目开发--bug mj157
- android 编写json 数据
- PopupWindow(自定义)
- 动态action的基本配置及实现
- 项目管理心得:一个项目经理的个人体会、经验总结
- i2c总线2
- SpringMVC 幂等请求