Spark Streaming 监控通过 nc -lk 9999 命令发送的word计数

来源:互联网 发布:CPDA数据分析师证书 编辑:程序博客网 时间:2024/05/15 05:59
package org.lm.spark.streamingimport org.apache.spark.SparkConfimport org.apache.spark.streaming.{Seconds, StreamingContext}object SparkStreamingQucikOps {  def main(args: Array[String]): Unit = {    val conf=new SparkConf().setAppName("Spark Streaming Qucik Ops").setMaster("spark://192.168.189.128:7077")    val ssc=new StreamingContext(conf,Seconds(5))    val lines=ssc.socketTextStream("192.168.189.129",9999)    val words=lines.flatMap(line=>line.split(" "))    val pairs=words.map(word=>(word,1))    val wordcount=pairs.reduceByKey(_+_)    wordcount.print()    ssc.start()    ssc.awaitTermination()  }}