分享一个hadoop例程

来源：互联网发布：python 当前毫秒数编辑：程序博客网时间：2024/05/18 11:35

功能：实现select sum(A.price),sum(B.count) form A B where A.id=B.id group by id;

数据源格式：

       A 1 2.4
       A 2 3.6
       A 1 4.8
       B 2 8
       B 2 4
       B 1 3

第一列为类型，相当于数据库中的表的性质，第二列为ID，第三列，如果是A，则为price，如果是B，则为count

程序：

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.conf.Configuration;

public class Test {
    public static class MyMapper extends Mapper<Object, Text, IntWritable, Text>{

        private Text word = new Text();

        public void map(Object key, Text value, Context context
                 ) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            IntWritable v = new IntWritable();
            while (itr.hasMoreTokens()) {
                String tmp = itr.nextToken();
                String t = itr.nextToken();
                String tt = itr.nextToken();
                //records in table A
                if(tmp.equals("A")){
                    word.set("Price "+tt);
                }else{                     //records in table B
                    word.set("Count "+tt);
                }
                int item = Integer.parseInt(t);
                v.set(item);
                context.write(v, word);
            }
        }
    }


    public static class MyReducer extends Reducer<IntWritable, Text,Text,Text>{

        public void reduce(IntWritable key,Iterable<Text> values, Context context)
                                                throws IOException, InterruptedException{
            int count = 0;
            float price = 0;
            String result = "";
            for(Text value:values){
                String v = value.toString();

                if(v.startsWith("Count")){
                    String [] cl = v.split(" ");
                    count = count + Integer.parseInt(cl[1]);
                }else{
                    String [] pl = v.split(" ");
                    price = price + Float.parseFloat(pl[1]);
                }

                result = "Sum(Count)="+count+"    Sum(Price)="+price;

            }
            String k = "ID = "+key.toString();
            context.write(new Text(k), new Text(result));
        }
    }

    public static void main(String [] argc) throws IOException, InterruptedException, ClassNotFoundException{
        if(argc.length!=2){
            System.out.println("Usage: test <input> <output>");
            System.exit(-1);
        }
        Configuration conf = new Configuration();
        Job job = new Job(conf,"test");
        job.setJarByClass(Test.class);
        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);

        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path(argc[0]));
        FileOutputFormat.setOutputPath(job, new Path(argc[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

}

进了步改时方案：多数情况下A 和 B应该来自不同的文件，所以在maper中应该先判断输入的value来自于文件A，还是文件B，我的下一步工作是改时这个程序，大家有兴趣可以一起来啊