Spark学习—统计文件单词出现次数

来源：互联网发布：怎么下载网络倾听者app 编辑：程序博客网时间：2024/06/01 09:38

上一节我们简单介绍了RDD中转化和执行操作的用法，本节将通过一个具体的示例来加深对RDD的认识。

一.需求

统计本地文件中单词出现次数

二.操作流程

1.读取外部文件创建JavaRDD;

2.通过flatMap转化操作切分字符串，获取单词新JavaRDD;

3.通过mapToPair，以key为单词，value统一为1的键值JavaPairRDD;

4.通过reduceByKey，累计叠加每个key，统计单词出现次数；

三.代码实现

package com.lm.sparkLearning.rdd;import java.util.Arrays;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.FlatMapFunction;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.PairFunction;import org.apache.spark.api.java.function.VoidFunction;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import com.lm.sparkLearning.utils.SparkUtils;import scala.Tuple2;/** * word文件单词出现个数计算 *  * @author dlm * */public class WordCountCalcLearing {private static Logger logger = LoggerFactory.getLogger(WordCountCalcLearing.class);public static void main(String[] args) {JavaSparkContext jsc = SparkUtils.getJavaSparkCOntext("WordCountSpark", "local[2]", "WARN");JavaRDD<String> wordRdd = SparkUtils.createRddExternal(jsc, "D:/README.txt");wordCountCal(wordRdd);jsc.stop();}/** * wordRdd统计计算逻辑 *  * @param wordRdd */public static void wordCountCal(JavaRDD<String> wordRdd) {// 将整个字符串根据空格分隔成单词JavaRDD<String> wordFlatMap = wordRdd.flatMap(new FlatMapFunction<String, String>() {/** *  */private static final long serialVersionUID = 1L;@Overridepublic Iterable<String> call(String t) throws Exception {return Arrays.asList(t.split("[^a-zA-Z']+"));}});// 将每个单词映射次数为1JavaPairRDD<String, Integer> wordMapToPair = wordFlatMap.mapToPair(new PairFunction<String, String, Integer>() {/** *  */private static final long serialVersionUID = 1L;@Overridepublic Tuple2<String, Integer> call(String t) throws Exception {return new Tuple2<String, Integer>(t, 1);}});// 将每个重复key的value相加JavaPairRDD<String, Integer> wordReduceByKey = wordMapToPair.reduceByKey(new Function2<Integer, Integer, Integer>() {/** *  */private static final long serialVersionUID = 1L;@Overridepublic Integer call(Integer v1, Integer v2) throws Exception {return v1 + v2;}});// 输出统计结果wordReduceByKey.sortByKey().foreach(new VoidFunction<Tuple2<String, Integer>>() {/** *  */private static final long serialVersionUID = 1L;@Overridepublic void call(Tuple2<String, Integer> t) throws Exception {logger.warn("key:" + t._1 + ",value:" + t._2);}});}}

四.下载代码

代码地址：http://download.csdn.net/detail/a123demi/9840519

OSChina : http://git.oschina.net/a123demi/sparklearning

0 0