1-2、自定义Partitioner代码

1-2、自定义Partitioner代码

1、输入数据：

hadoop|hello

spark|which

spark|who

storm|however

hadoop|code

spark|water[

HDFS上：

2、代码：

package 自定义分区;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Partitioner;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

/**

 * main

 * @author Administrator

 *

 */

public class PartitionerDemo {

 public static void main(String[] args) throws Exception {

  Configuration conf=new Configuration();

  Job job=Job.getInstance(conf, PartitionerDemo.class.getSimpleName());

 

  job.setJarByClass(PartitionerDemo.class);

  job.setNumReduceTasks(3);

 

  job.setMapperClass(MyMapper.class);

  job.setReducerClass(MyReduce.class);

 

  job.setPartitionerClass(MyPartitionerPar.class);

 // job.setCombinerClass(MyReduce.class);

 

  job.setInputFormatClass(TextInputFormat.class);

  job.setOutputFormatClass(TextOutputFormat.class);

 

  job.setMapOutputKeyClass(Text.class);

  job.setMapOutputValueClass(Text.class);

 

  job.setOutputKeyClass(Text.class);

  job.setOutputValueClass(Text.class);

 

  FileInputFormat.setInputPaths(job, "hdfs://192.168.1.10:9000"+args[0]);

  FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.1.10:9000"+args[1]));

 

  job.waitForCompletion(true);

 }

 /**

  * map

  * @author Administrator

  *

  */

 public static class MyMapper extends Mapper<LongWritable, Text, Text, Text>{

  Text v2=new Text();

  Text k2=new Text();

  @Override

  protected void map(LongWritable key, Text value,

    Mapper<LongWritable, Text, Text, Text>.Context context)

    throws IOException, InterruptedException {

   

   String line=value.toString();

   String[] splited=line.split("\\|");

   k2.set(splited[0]);

   v2.set(splited[1]);

   context.write(k2, v2);

   System.out.println(splited[0]+"-->"+splited[1]);

  }

 }

 

 /**

  * reducer

  * @author Administrator

  *

  */

 public static class MyReduce extends Reducer<Text, Text, Text, Text>{

  Text v3=new Text();

  @Override

  protected void reduce(Text k2, Iterable<Text> v2,

    Reducer<Text, Text, Text, Text>.Context context)

    throws IOException, InterruptedException {

   for (Text str : v2) {

    v3.set(str);

    context.write(k2, v3);

   }

  }

 }

 

 

 /**

  * 输入的是Map的输出

  * @author Administrator

  *

  */

 public static class MyPartitionerPar extends Partitioner<Text, Text> {

  @Override

  public int getPartition(Text key, Text value, int numPartitions) {

   String str1=key.toString();

   int result=1;

   if(str1.equals("hadoop")){

    result = 0 % numPartitions;   //0%3==0

// result = 0 ;   //0%3==0

   }else if(str1.equals("storm")){

    result = 1 % numPartitions;   //0%3==0

// result = 1 ; //1%3==1

   }else if (str1.equals("spark")) {

    result = 2 % numPartitions;   //0%3==0

// result = 2 ; //2%3==2

   }

   return result;

  }

 }

 

}

自定义分区关键代码：

public static class MyPartitionerPar extends Partitioner<Text, Text> {

  @Override

  public int getPartition(Text key, Text value, int numPartitions) {

   String str1=key.toString();

   int result=1;

   if(str1.equals("hadoop")){

    result = 0 % numPartitions;   //0%3==0

// result = 0 ;   //0%3==0

   }else if(str1.equals("storm")){

    result = 1 % numPartitions;   //0%3==0

// result = 1 ; //1%3==1

   }else if (str1.equals("spark")) {

    result = 2 % numPartitions;   //0%3==0

// result = 2 ; //2%3==2

   }

   return result;

  }

3、输出结果：

part-r-00000：

part-r-00001：

part-r-00002：