Spark之BroadCast
来源:互联网 发布:钱永健 知乎 编辑:程序博客网 时间:2024/05/22 00:12
package com.uplooking.bigdata.core.p3;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
import java.util.Arrays;
import java.util.List;
/**
* Spark使用广播变量进行计算
*/
public class JavaSparkBroadCastApp {
public static void main(String[] args) {
if(args == null || args.length < 1) {
System.err.println("Usage :<times[Integer]>");
System.exit(-1);
}
SparkConf conf = new SparkConf();
conf.setAppName(JavaSparkBroadCastApp.class.getSimpleName());
conf.setMaster("local[2]");
JavaSparkContext sc = new JavaSparkContext(conf);
List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
JavaRDD<Integer> listRDD = sc.parallelize(list);
int times = Integer.valueOf(args[0].trim());
//设置一个广播变量来解决
Broadcast<Integer> timesBC = sc.broadcast(times);
//将集合中的每一个元素*times
JavaRDD<Integer> retRDD = listRDD.map(ele -> {
return ele * timesBC.value();
});
retRDD.foreach(ele -> System.out.println(ele));
}
}
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
import java.util.Arrays;
import java.util.List;
/**
* Spark使用广播变量进行计算
*/
public class JavaSparkBroadCastApp {
public static void main(String[] args) {
if(args == null || args.length < 1) {
System.err.println("Usage :<times[Integer]>");
System.exit(-1);
}
SparkConf conf = new SparkConf();
conf.setAppName(JavaSparkBroadCastApp.class.getSimpleName());
conf.setMaster("local[2]");
JavaSparkContext sc = new JavaSparkContext(conf);
List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
JavaRDD<Integer> listRDD = sc.parallelize(list);
int times = Integer.valueOf(args[0].trim());
//设置一个广播变量来解决
Broadcast<Integer> timesBC = sc.broadcast(times);
//将集合中的每一个元素*times
JavaRDD<Integer> retRDD = listRDD.map(ele -> {
return ele * timesBC.value();
});
retRDD.foreach(ele -> System.out.println(ele));
}
}
0 0
- Spark之BroadCast
- Spark 之Broadcast
- Spark Broadcast之TorrentBroadcast
- spark-broadcast
- Spark源码阅读笔记之Broadcast(一)
- spark之广播共享变量broadcast
- Spark大师之路:广播变量(Broadcast)源码分析
- Spark学习之6:Broadcast及RDD cache
- Spark源码阅读笔记之Broadcast(二)
- Spark源码阅读笔记之Broadcast(三)
- spark join broadcast优化
- Spark Broadcast源码分析
- Spark Broadcast内幕解密
- Spark Broadcast源码分析
- spark join broadcast优化
- Spark Broadcast 广播变量
- Spark Broadcast内幕分析
- spark Broadcast 内幕解密
- Windows 服务状态在启动或停止时卡住的解决办法
- 移动端吸顶、动画、浏览器层模型以及相关总结
- Myeclips导出war错误
- fmdb实现sqlite数据库的增删改查功能(下)
- 京东——分堆
- Spark之BroadCast
- MySQL中varchar 的最大长度
- C语言中 关于const和#define的区别心得
- 疑问+点子+理解
- 【JVM学习】之 ClassLoader
- Spark之二次排序
- git基础使用命令记录
- 无向图的连通性问题(DFS)
- 笔记本电脑开启无线网络连接功能