Spark Streaming DSstream 的updateByKey用法

来源：互联网发布：iphone6蜂窝数据4g设置编辑：程序博客网时间：2024/05/22 12:32

package org.lm.spark.streamingimport org.apache.spark.streaming.{Seconds, StreamingContext}import org.apache.spark.{SparkConf, SparkContext}object StatefulWordCountOnLine {  def main(args: Array[String]): Unit = {

//定义updateByKey使用到的函数    val updateFunc=(values:Seq[Int],state:Option[Int])=>{      val currentCount=values.foldLeft(0)(_+_)      val previousCount=state.getOrElse(0)      Some(currentCount+previousCount)    }    val conf=new SparkConf().setAppName("Stateful Word Count On Line").setMaster("spark://192.168.189.128:7077")    val sc=new SparkContext(conf)    val ssc=new StreamingContext(sc,Seconds(5))

//定义持久化存储位置，很重要，必须定义    ssc.checkpoint("hdfs://192.168.189.128:9000/user/input/StatefulWordCount/log")    val lines=ssc.socketTextStream("192.168.189.129",9999)    val words=lines.flatMap(_.split(" "))    val pairs=words.map(word=>(word,1))

//调用updateStateByKey方法，参数传入前面定义的updateFunc函数    val statefulDStream=pairs.updateStateByKey[Int](updateFunc)    statefulDStream.print()    ssc.start()    ssc.awaitTermination()  }}

阅读全文

0 0