hadoop基本操作

来源：互联网发布：linux gid是什么意思编辑：程序博客网时间：2024/06/13 21:25

hadoop的基本操作是对6个类的重写实现的

TextInputFormat，Mapper，Combiner，HashPartitioner，Reducer，TextOutFormat

//基本的设置，对于同的问题，只需要重写6个类即可</span>

public static void test2() throws Exception {Configuration conf = new Configuration();String arg1 = "hdfs://101.200.200.114:9008/yj/input";String arg2 = "hdfs://101.200.200.114:9008/yj/output";Job job = new Job(conf, "word count");job.setJarByClass(WordCount.class);//对hadoop的操作基本上是通过对下面6个类重写实现的job.setInputFormatClass(TextInputFormat.class);//<span style="font-family: Arial, Helvetica, sans-serif; font-size: 12px;">TextInputFormat为默认，将输入文件分割成小文件，解析成<key,value>对，key默认为字符偏移量，value默认为行值</span>job.setMapperClass(MyMapper.class);job.setCombinerClass(IntSumReducer.class);job.setPartitionerClass(HashPartitioner.class);//<span style="font-family: Arial, Helvetica, sans-serif; font-size: 12px;">HashPartitioner为默认</span>job.setReducerClass(MyReducer.class);job.setOutputFormatClass(TextOutFormat.class);//<span style="font-family: Arial, Helvetica, sans-serif; font-size: 12px;">TextOutFormat为默认</span>FileInputFormat.addInputPath(job, new Path(arg1));FileOutputFormat.setOutputPath(job, new Path(arg2));System.exit(job.waitForCompletion(true) ? 0 : 1);}

//统计单词个数public static void test3() throws Exception {Configuration conf = new Configuration();String arg1 = "hdfs://101.200.200.114:9008/yj/input";String arg2 = "hdfs://101.200.200.114:9008/yj/output";Job job = new Job(conf, "word count");job.setJarByClass(WordCount.class);job.setInputFormatClass(TextInputFormat.class);//使用默认job.setMapperClass(TokenizerMapper.class);//将每行分割成<word,1>的格式传给下一个过程job.setCombinerClass(null);//将map后的相同key先进行一个合并，增加这个过程可以减少io，提高速度job.setPartitionerClass(HashPartitioner.class);//shuffle过程中将数据分给不同reducer的策略，默认使用hashjob.setReducerClass(IntSumReducer.class);//合并中间结果job.setOutputKeyClass(Text.class);//由于输出的每行包含两个，分别是key和value，故需如此设置，也可重写OutFormatjob.setOutputValueClass(IntWritable.class);FileInputFormat.addInputPath(job, new Path(arg1));FileOutputFormat.setOutputPath(job, new Path(arg2));System.exit(job.waitForCompletion(true) ? 0 : 1);}

0 0