9。24 mapreduce简单实例练习

JOB


public class Tranconfig {
            public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
String properties=args[0];
String inputpath=args[1];
String outputpath=args[2];
                 Configuration conf = new Configuration();
                 ConstUtil.init(conf,properties); //初始化配置文件
                 //将参数设置到config中
                 conf.set("job_name", conf.get(ConstUtil.HADOOP_JOB_NAME));
                 conf.set("zookeeper", conf.get(ConstUtil.ZOOKEEPER_ZNODE_PARENT));
                 conf.set("reduce_num", conf.get(ConstUtil.HADOOP_REDUCE_TASKS));
                 Job job = Job.getInstance(conf,conf.get("job_name"));
                 job.setJarByClass(Tranconfig.class);//设置job启动的主类
                 job.setMapperClass(TranConfigMap.class);//添加map类
                 job.setOutputKeyClass(Text.class);//设置KEY输出的类型
                 job.setOutputValueClass(Text.class);//设置value输出的类型
                 job.setNumReduceTasks(1);
                 FileInputFormat.addInputPath(job, new Path(inputpath));
                 FileOutputFormat.setOutputPath(job, new Path(outputpath));
                 job.waitForCompletion(true);
}
}


public class TranConfigMap extends Mapper<LongWritable,Text,Text,Text>{
    private String job_name;
    private String reduce_num;
    private String zookeeper;
    
protected void setup(Mapper<LongWritable,Text,Text,Text>.Context context) 
     throws IOException ,InterruptedException {
              Configuration prop = context.getConfiguration();
              job_name=prop.get("job_name");
              reduce_num=prop.get("reduce_num");
              zookeeper=prop.get("zookeeper");
              super.setup(context);
      }
      protected void map(LongWritable key, Text value, Mapper<LongWritable,Text,Text,Text>.Context context) 
     throws IOException ,InterruptedException {
         String line=value.toString();
         if(line.contains("hadoop")){
         context.write(new Text("hadoop"), new Text(job_name));
         }
         if(line.contains("zookeeper")){
         context.write(new Text("zookeeper"), new Text(zookeeper));
         }
         if(line.contains("reduce_num")){
         context.write(new Text("reduce_num"), new Text(reduce_num));
         }
         
      }
}


代码编译完后export项目 选择java file 
连接本地hadoop伪分布式  先将导出来的jar包和config.properties传到本地   之后根据代码在hadoop服务器上建立input文件夹，然后vim demo.txt创建需要访问的文件       
通过类似于 jar Tranconfig.jar（导出的jar包） com.zfh.job.Tranconfig（job主类名） config.properties /user/input/demo.txt /user/output/res的命令执行mapredeuce
执行后进入res目录下  通过cat命令查看part-r-00000日志 






Pv统计


public class PvUvJob {
   public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
Job job =Job.getInstance(conf,"Pv");//创建一个实例 名字叫pv
job.setJarByClass(PvUvJob.class);
Path in=new Path("/user/input/pv");
Path out=new Path("/user/output/pv");
FileInputFormat.addInputPath(job, in);
FileOutputFormat.setOutputPath(job, out);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setMapperClass(PvUvMap.class);
job.setReducerClass(PvUvReduce.class);
//map输出类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
    //reduce输出类型
job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);
    job.setNumReduceTasks(1);
    //提交job
    job.waitForCompletion(true);
    }
}
 
 public class PvUvMap extends Mapper<LongWritable, Text, Text, IntWritable>{
 IntWritable one = new IntWritable(1);
           protected void map(LongWritable key, Text value, Mapper<LongWritable,Text,Text,IntWritable>.Context context) 
          throws java.io.IOException ,InterruptedException {
          String []line=value.toString().split("\t");
          String url=line[28];
          context.write(new Text(url),one);    
           }
}


public class PvUvReduce extends Reducer<Text ,IntWritable, Text, NullWritable> {
            protected void reduce(Text key, Iterable<IntWritable> values, 
            Reducer<Text,IntWritable,Text,NullWritable>.Context context) throws java.io.IOException ,InterruptedException {
            int sum=0;
            for(IntWritable value:values){
            sum+=value.get();
            }
            context.write(new Text(key+"====>"+sum), NullWritable.get());//空输出 NullWritable.get()
            }
            
}