Scala 版 word count

来源：互联网发布：淘宝怎么搜原味胖次编辑：程序博客网时间：2024/05/19 17:04

package com.oracle.scalaimport org.apache.spark.SparkConfimport org.apache.spark.SparkContext/** * WordCount示例 */object WordCount {  def main(args: Array[String]): Unit = {    //创建SparkConf Spark运行时配置    val sparkConf = new SparkConf().setAppName("WordCount")      .setMaster("local")           //通过传入的SparkConf配置对象实例化一个SparkContext      val sc = new SparkContext(sparkConf)        //从本地文件加载数据产生RDD    val linesRdd = sc.textFile("C:\\Users\\Desktop\\csd\\wordcount.txt")        //将每一行的记录通过指定分隔符分隔    val wordsRdd =  linesRdd.flatMap { _.split(" ") }        //将每一个单词设置为1  形成key-value    val pairRdd =wordsRdd.map((_ , 1))        //相同的key的value进行聚合    val resultRdd = pairRdd.reduceByKey(_+_)        //触发Job   val results = resultRdd.collect()      for(result <- results){     println(result)   }    sc.stop()  }}

0 0