利用mapreduce统计单词个数

wordCount.class

package wordCount;import java.io.IOException;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class wordCount {    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        // TODO Auto-generated method stub        Configuration configuration = new Configuration();        FileSystem fs = FileSystem.get(configuration);        Job job = new Job(configuration);        job.setJarByClass(wordCount.class);        FileInputFormat.addInputPath(job,new Path("/file"));        if(fs.exists(new Path("/result"))){            System.out.println("文件已存在，将删除后重建！");            fs.delete(new Path("/result"));        }        FileOutputFormat.setOutputPath(job,new Path("/result"));        job.setMapperClass(myMapper.class);        job.setReducerClass(myReducer.class);        job.setNumReduceTasks(2);        job.setPartitionerClass(myPartitioner.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(IntWritable.class);        job.waitForCompletion(true);    }}

myMapper.class

package wordCount;import java.io.IOException;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Mapper;public class myMapper extends Mapper<LongWritable, Text,Text, IntWritable>{    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException{        //String[] str = value.toString().split("[^a-zA-Z0-9]");        String[] str = value.toString().split(" ");        Text ss = new Text();        /*for(String ss:str){            if(!ss.equals(" ")&&ss!=null){*/                //context.write(new Text(ss),new IntWritable(1));                ss.clear();                ss.set(new StringBuffer("loglevel::").append(str[1]).toString());                context.write(ss, new IntWritable(1));                ss.clear();                ss.set(new StringBuffer("logresource::").append(str[2]).toString());                context.write(ss, new IntWritable(1));        /*  }        }*/    }}

myPartitioner.class

package wordCount;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Partitioner;public class myPartitioner extends Partitioner<Text, IntWritable> {    @Override    public int getPartition(Text key, IntWritable value, int arg2) {        // TODO Auto-generated method stub        if(key.toString().startsWith("loglevel::"))            return 0;        if(key.toString().startsWith("logresource::"))            return 1;        return 0;    }}

myReducer.class

package wordCount;import java.io.IOException;import java.util.Iterator;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Reducer;public class myReducer extends Reducer<Text,IntWritable, Text,IntWritable>{    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException{        int sum = 0;        Iterator<IntWritable> it = values.iterator();        while(it.hasNext()){            sum += it.next().get();        }        context.write(key, new IntWritable(sum));    }}