Spark Streaming---PersistMySQL Work Count(java)
来源:互联网 发布:在淘宝上怎么买三唑仑 编辑:程序博客网 时间:2024/06/03 21:34
package com.spark.streaming;import java.sql.Connection;import java.sql.Statement;import java.util.Arrays;import java.util.Iterator;import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.function.FlatMapFunction;import org.apache.spark.api.java.function.Function2;import org.apache.spark.api.java.function.PairFunction;import org.apache.spark.api.java.function.VoidFunction;import org.apache.spark.streaming.Durations;import org.apache.spark.streaming.api.java.JavaDStream;import org.apache.spark.streaming.api.java.JavaPairDStream;import org.apache.spark.streaming.api.java.JavaStreamingContext;import scala.Tuple2;public class PersistMySQLWordcount { public static void main(String[] args) { SparkConf conf = new SparkConf().setMaster("local[1]").setAppName("PersistMySQLWordcount"); JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5)); JavaDStream<String> lines = jssc.textFileStream("hdfs://node12:8020/Spark/Streaming/WordCount"); JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() { private static final long serialVersionUID = 1L; @Override public Iterable<String> call(String line) throws Exception { return Arrays.asList(line.split(" ")); } }); JavaPairDStream<String, Integer> pairs = words.mapToPair(new PairFunction<String, String, Integer>() { private static final long serialVersionUID = 1L; @Override public Tuple2<String, Integer> call(String word) throws Exception { return new Tuple2<String,Integer>(word, 1); } }); JavaPairDStream<String, Integer> wordcounts = pairs.reduceByKey(new Function2<Integer, Integer, Integer>() { private static final long serialVersionUID = 1L; @Override public Integer call(Integer v1, Integer v2) throws Exception { return v1 + v2; } }); wordcounts.print(); wordcounts.foreachRDD(new VoidFunction<JavaPairRDD<String, Integer>>() { private static final long serialVersionUID = 1L; @Override public void call(JavaPairRDD<String, Integer> wordcountsRDD) throws Exception { wordcountsRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String,Integer>>>() { private static final long serialVersionUID = 1L; @Override public void call(Iterator<Tuple2<String, Integer>> wordcounts) throws Exception { Connection conn = ConnectionPool.getConnection(); Tuple2<String, Integer> wordcount = null; while(wordcounts.hasNext()){ wordcount = wordcounts.next(); String sql = "insert into wordcount(word,count) " + "values('" + wordcount._1 + "'," + wordcount._2 + ")"; Statement stmt = conn.createStatement(); stmt.executeUpdate(sql); } ConnectionPool.returnConnection(conn); } }); } }); jssc.start(); jssc.awaitTermination(); jssc.close(); }}
阅读全文
1 0
- Spark Streaming---PersistMySQL Work Count(java)
- Spark Streaming---Streaming Word Count(java)
- Spark Streaming---UpdatestateBykey(java)
- Spark Streaming---WindowBasedTopWord(java)
- Spark Streaming Java案例
- Spark Streaming----Transform Operation(java)
- Spark Streaming和Flink的Word Count对比
- spark-streaming 编程(二) word count单词计数统计
- Spark Streaming
- spark streaming
- Spark/Streaming
- Spark Streaming
- spark streaming
- Spark Streaming
- Spark Streaming
- Spark Streaming
- Spark Streaming
- spark streaming
- 剑指Offer面试题49字符串转整数,面试题50二叉树两个结点的最低公共祖先
- poj 2762 Going from u to v or from v to u? 单向连通图判定
- smb安装脚本解释
- 求数组中的最大值和下标
- js+插件实现代码复制及动态生成二维码扫描、分享到朋友圈QQ空间功能
- Spark Streaming---PersistMySQL Work Count(java)
- (hdu 1060 ,1061)求N^N的最高位和最低位(取对数,快速幂)
- MySQL性能分析及explain的使用
- ice 3.6.3安装
- Windows平台分布式架构实践
- 利用fsockopen模拟HTTP发送请求
- Java泛型理解
- Java发送邮件
- Appium使用命令行方式启动服务