sparksession

来源：互联网发布：经济型酒店数据分析编辑：程序博客网时间：2024/04/26 01:26
package sparksqlimport org.apache.spark.SparkConfimport org.apache.spark.sql.{Row, SparkSession}/**  * Created by zengxiaosen on 16/9/20.  */case class Log01(id01: Int, content01: String)object sparksessiontest {  def main(args: Array[String]): Unit = {    val sparkConf = new SparkConf().setAppName("sparksessiontest").setMaster("local")    val ss = SparkSession.builder()        .config(sparkConf)        .getOrCreate()    val sc = ss.sparkContext    import ss.implicits._    val rdd = sc.parallelize((0 to 99).map(i => Log01(i, s"content_$i")))    val rdd2 = ss.createDataFrame((0 to 99).map(i => Log01(i, s"content_$i"))).toDF("rid","name")    //临时视图    //rdd.toDF("id","content").createTempView("log")    rdd2.createOrReplaceTempView("log01")    val top = 5    val sql = "select * from log01 where rid <=" + top    //多行的语法    val sql01 =      s"""         |select * from log01         |where rid <=$top         |and 1=1       """.stripMargin    //val result = ss.sql("select * from log01")    val result = ss.sql(sql01)    result.show()    result.printSchema()    result.rdd.map{ row =>      val id = row.getAs[Int]("rid")      val content = row.getAs[String]("name")      (id, content)    }.foreach(println)    println("==============================")    result.rdd.map{ row =>    val id = row.getInt(0)    val content = row.getString(1)      (id, content)    }.foreach(println)    //拼成字符串    result.rdd.map{      case Row(mid: Int, mName: String) => (s"$mid", s"$mName")    }foreach(println)    sc.stop()    ss.stop()  }}
0 0