spark的streamingcontext的另一种创建方式

来源:互联网 发布:淄博阿雷seo推广 编辑:程序博客网 时间:2024/05/21 11:17
val ssc = StreamingContext.getOrCreate(checkpointDirectory,  () => {    val topics = "logTopic"    val brokers = "master:9092,slave1:9092"    val sparkConf = new SparkConf().setAppName("uv").setMaster("local[2]")    val ssc = new StreamingContext(sparkConf, Seconds(2))    ssc.checkpoint(checkpointDirectory)    //create direct kafka stream with brokers and topics    /*    直接消费是不经过zookeeper的,所以这时候你就要告诉我kafka的地址,而不是zookeeper的地址里     */    val topicSet = topics.split(",").toSet    val kafkaParams = Map[String, String]("metadata.broker.list" -> brokers)    //stringdecoder表示传的消息是字符串类型的    val messages = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](      ssc, kafkaParams, topicSet    )    computeUV(messages)    ssc  })
0 0