MapReduce的运行

来源：互联网发布：天启小米抢购软件编辑：程序博客网时间：2024/05/16 05:46

Map Reduce操作步骤：

1.启动hadoop集群 ./startall.sh
2.进入jar包所在文件夹Demo
3.在hadoop新建 input hadoop fs -mkdir -p /user/input
4.把分析文件put在input中 hadoop fs -put demo.txt /user/input
5. Hadoop jar TranConfig.jar com.job.Tranconfig config.properties
/ user/input/demo.txt /user/output/res
6.Hadoop jar jar包名包.类.方法 /user/input/分析文件/user/output/Pv

Hadoop实例

PvUvJob

package com.map;import java.io.IOException;import java.util.Map;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class PvUvMap extends Mapper<LongWritable, Text, Text, IntWritable>{    IntWritable one = new IntWritable(1);       protected void map(LongWritable key, Text value,Context context)            throws IOException, InterruptedException {        String []res=value.toString().split("\t");        String url=res[28];        context.write(new Text(url),one);                super.map(key, value, context);    }}

PvUvmap

package com.map;import java.io.IOException;import java.util.Map;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class PvUvMap extends Mapper<LongWritable, Text, Text, IntWritable>{    IntWritable one = new IntWritable(1);       protected void map(LongWritable key, Text value,Context context)            throws IOException, InterruptedException {        String []res=value.toString().split("\t");        String url=res[28];        context.write(new Text(url),one);                super.map(key, value, context);    }}

PvUvReduce

package com.reduce;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.NullWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;public class PvUvReduce extends Reducer<Text,IntWritable,Text,NullWritable> {    protected void reduce(Text key,Iterable<IntWritable>values,Context context)    throws IOException,InterruptedException{        int sum = 0;        for(IntWritable value:values){            sum+=value.get();        }        context.write(new Text(key.toString()+":"+sum), NullWritable.get());    }}

阅读全文

0 0