MapReduce 程序（1）

来源：互联网发布：网络电视机排名编辑：程序博客网时间：2024/05/18 03:34

之前在网上看了很多MapReduce程序计数的帖子，在这里跟大家分享一下自己编写的代码。

此文章适合初入门级别。

file1:

sunying 89

yangkun 95

wangjie 75

zhaode 85

xiaoming 90

file2:

sunying 85

yangkun 75

wangjie 79

zhaode 89

xiaoming 88

file3:

sunying 78

yangkun 90

wangjie 85

zhaode 80

xiaoming 93

package org.sunying.practice;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class AvgScore {


   public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

           Configuration conf = new Configuration();

           Job job = Job.getInstance(conf);

           job.setJarByClass(AvgScore.class);

           // 设置map类，定义输出的数据类型
           job.setMapperClass(ASMapper.class);
           job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(LongWritable.class);

           // 设置reduce类，定义输出的数据类型
           job.setReducerClass(ASReducer.class);
           job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);

           // 设置输入输出的文件目录
           FileInputFormat.setInputPaths(job, new Path("D://hadoop-2.6.0//input//score"));
           FileOutputFormat.setOutputPath(job, new Path("D://hadoop-2.6.0//output"));

           // 这几行代码是清理输出结果的文件
           Path path = new Path("D://hadoop-2.6.0//output");
           FileSystem fileSystem = path.getFileSystem(conf);
           if(fileSystem.exists(path)){
               fileSystem.delete(path, true);// true表示文件里有内容时也删除文件。
           }

           job.waitForCompletion(true);

   }

   public static class ASMapper extends Mapper<LongWritable, Text, Text, LongWritable>{

       @Override
       protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
           // 将读入的内容转成string类型并分割，输出到reduce
           String[] str = value.toString().split("\t");
           context.write(new Text(str[0]), new LongWritable(Long.parseLong(str[1])));
       }

   }


   public static class ASReducer extends Reducer<Text, LongWritable, Text, Text>{

       @Override
       protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
           long count = 0;
           for (LongWritable l : values) {
                 count = count + l.get();
           }
           // 此处将结果四舍五入，保留两位小数。这是简单的方法。
           double rr =count / 3.0;
           String ff = String.format("%.2f", rr);

           context.write(key, new Text(ff));

       }

   }



}

阅读全文

1 0