WorldCount程序分析
来源:互联网 发布:windows激活密匙在哪里 编辑:程序博客网 时间:2024/05/16 10:29
package hadoop;/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */#加载java的io抛错类import java.io.IOException;#字符串按令牌格式化类import java.util.StringTokenizer;#读配置参数类import org.apache.hadoop.conf.Configuration;#文件系统路径类import org.apache.hadoop.fs.Path;#整数的WritableComparable类import org.apache.hadoop.io.IntWritable;#存储字符串类import org.apache.hadoop.io.Text;#job类import org.apache.hadoop.mapreduce.Job;#mapper类import org.apache.hadoop.mapreduce.Mapper;#reducer类import org.apache.hadoop.mapreduce.Reducer;#文件InputFormat类import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;#文件OutputFormat类import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;#输入参数解析类import org.apache.hadoop.util.GenericOptionsParser;public class WordCount { #实现自己的TokenizerMapper类,继承自Mapper public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable>{ #创建one整数1实例 private final static IntWritable one = new IntWritable(1); #创建Text的实例word private Text word = new Text(); #重写父类Mapper类的map方法 public void map(Object key, Text value, Context context ) throws IOException, InterruptedException { #对输入值value按令牌方式StringTokenizer StringTokenizer itr = new StringTokenizer(value.toString()); #循环有tokens while (itr.hasMoreTokens()) { #循环获取下一个token word.set(itr.nextToken()); #写入word和one数量 context.write(word, one); } } } public static class IntSumReducer extends Reducer<Text,IntWritable,Text,IntWritable> { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable<IntWritable> values, Context context ) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); } result.set(sum); context.write(key, result); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2); } //conf.set("fs.defaultFS", "hdfs://192.168.6.77:9000"); Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }}
0 0
- WorldCount程序分析
- Hadoop WorldCount程序
- worldcount程序java版本
- Hadoop学习笔记(七)---简单WorldCount程序的实现
- hadoop学习4--MapReduce及官方WorldCount分析
- WorldCount运行讲解
- SparkStreaming的worldCount实例
- spark: scala实现worldcount
- cloudcomputing.ruc题目1000:WorldCount
- 使用Hadoop运行WorldCount示例
- hadoop 运行WorldCount时出错,找不到主程序
- 在spark下用pyhton写worldCount
- Mac下使用Spark-shell实现WorldCount
- Hadoop之——WorldCount统计实例
- spark Streaming编写一个WorldCount的例子
- 用eclipse 连接kafka streaming 执行worldcount
- 程序分析
- 程序分析
- 用js获取url各部分参数
- ubuntu14.04 u盘安装教程
- 比树莓派配置好接地气的香蕉派上手初体验
- 如何检测浏览器是否支持HTML5
- 去掉手机 Update Devices Infomation
- WorldCount程序分析
- 仿照QQ抖一抖的实现——窗口抖动功能
- PAT 1064. Complete Binary Search Tree
- 百度实时推送代码 curl
- java transient简介
- 同义词的循环链
- 图像差分 (IplImage *) 版
- usb 枚举
- java学习笔记(二) -- 基本语法