hadoop之 mapreduce

来源：互联网发布：恐怖主义数据库编辑：程序博客网时间：2024/06/06 00:48

直接给上代码

package com.hdfs;

import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class MapTest {
public static class MyMap extends
Mapper<LongWritable, Text, Text, IntWritable> {

@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
System.out.println("map------------->>>>");
String line = value.toString();
String[] s = line.split(",");
String year = s[4];
String years=year.substring(0,4);
int money = Integer.parseInt(s[2]);
context.write(new Text(years), new IntWritable(money));
}

}

public static class MyReduce extends
Reducer<Text, IntWritable, Text, IntWritable> {

@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
System.out.println("reduce----------->>>");
int sum = 0;
int count = 0;
Iterator<IntWritable> iterator = values.iterator();
while (iterator.hasNext()) {
sum += iterator.next().get();
count++;
}
int avg = (int) sum / count;
context.write(key, new IntWritable(avg));

}

}

public static void main(String[] args) throws IOException,
ClassNotFoundException, InterruptedException {
Job job = new Job();
job.setJarByClass(MapTest.class);
job.setJobName("moneyAvg");
FileInputFormat.addInputPath(job, new Path("/data/example/shen1"));
FileOutputFormat.setOutputPath(job, new Path("/data/example/shen11"));
job.setMapperClass(MyMap.class);
job.setReducerClass(MyReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
System.exit(job.waitForCompletion(true) ? 0 : 1);

}
}

数据测试的类型如下：

1741-1820-1767-573,公良华灿,13,男,20160211,5898.0,CelineDion、金橘柠檬、九龙液酒
1791-1896-1825-556,宣冠宇,41,男,20151231,3290.0,菠萝汁、JaneIredale珍爱芮德
1130-1332-1852-145,桑磊,12,男,20151209,7471.0,NapoleonPerdis

0 0