scala实例——五
来源:互联网 发布:开源游戏服务器源码 编辑:程序博客网 时间:2024/06/10 02:59
import scala.util.control.Breaks._import org.apache.spark.sql._import org.apache.spark.SparkConfimport org.apache.spark.SparkContextimport org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}import org.apache.spark.rdd.RDDimport org.apache.spark.sql.execution.vectorized.ColumnarBatch.Rowimport scala.util.Randomimport scala.util.control.Breaksobject come_in_analyse { def main(args: Array[String]) { val spark = SparkSession .builder() .appName("come_in_analyse") .config("spark.some.config.option", "some-value") .getOrCreate() import spark.implicits._ val df1 = spark.read.json("/spark_data/visit_records.json") df1.createOrReplaceTempView("visit") val df2 = spark.read.json("/spark_data/people_flow.json") df2.createOrReplaceTempView("people_flow") // Global temporary view is tied to a system preserved database `global_temp` var result_string = "" var sql = "SELECT in_time from visit order by `in_time` limit 1" var min_time = (spark.sql(sql).collect()) (0).asInstanceOf[Row].getInt(0) sql = "SELECT in_time from visit order by `in_time` desc limit 1 " var max_time = (spark.sql(sql).collect()) (0).asInstanceOf[Row].getInt(0) var now_time = 0; var outer = new Breaks now_time = min_time; while (now_time <= max_time) { outer.breakable { var come_num = 0; var time1 = now_time; var time2 = now_time + 60; var sql2 = "SELECT count(*) num from visit where `in_time` >= " + time1 + " and `in_time` <= " + time2 + ""; //SQL语句 come_num = (spark.sql(sql2).collect()) (0).asInstanceOf[Row].getInt(0) if (come_num == 0) { now_time = now_time + 60; outer.break; } var sql3 = "SELECT num from people_flow where `time` = " + time1 + ""; //SQL语句 var people_flow_num = (spark.sql(sql3).collect()) (0).asInstanceOf[Row].getInt(0) var in_rate = (come_num.asInstanceOf[Float]/people_flow_num.asInstanceOf[Float]) var format_in_rate = f"$in_rate%1.2f" //每一条 jump 结果 添加到 结果集 var in_string = """{"time":"""+time1 +","+""""num":"""+come_num+","+""""in_rate":"""+format_in_rate+"}\n" result_string = result_string + in_string now_time = now_time + 60; } } //将结果集 存入 文件 import java.io._ val writer = new PrintWriter(new File("\\sparkdata\\come_in_shop.json" )) writer.write(result_string) writer.close() }
0 0
- scala实例——五
- scala 实例——一
- scala实例——二
- scala实例——三
- scala实例——四
- scala实例——六
- scala——实例 new_customer_extract
- scala 代码实例——customer_extract
- Scala进阶源码实战之五——List、Queue、Stack、Set、Map
- scala 学习(五)——def和procedure的区别和使用
- 《机器学习实战(Scala实现)》(五)——Logistic回归
- [scala之五]对象
- Scala练习(五)
- scala学习笔记五
- Scala学习笔记(五)
- 五.Scala变量
- Scala学习五:异常
- scala —— maven scala项目开发
- 程序
- scala实例——四
- StringBuilder的append()效率比String的+运算符效率高太多
- LeetCode127 Word Ladder
- Java 中的静态内部类
- scala实例——五
- LinkedIn的高可用消息系统Kafka应用介绍
- Java Itext+jfreehart 导入含有图形的word文档
- BIOS和DOS中断处理(1302)
- C#集合
- C语言程序设计(15)
- 霍夫变换
- EOJ-大学生程序设计邀请赛(华东师范大学)-E-黑心啤酒厂
- 【iOS沉思录】BAD_ACCESS内存错误调试