Spark注册UDF函数，用于DataFrame DSL or SQL

网址：https://www.cnblogs.com/wwxbi/p/6798384.html

import org.apache.spark.sql.SparkSession

import org.apache.spark.sql.functions._

 

object Test2 {

  def main(args: Array[String]): Unit = {

    val spark = SparkSession

      .builder()

      .appName("Spark SQL basic example")

      .config("spark.some.config.option", "some-value")

      .getOrCreate()

 

    // For implicit conversions like converting RDDs to DataFrames

    import spark.implicits._

 

    val df = Seq(("id1", 1, 100), ("id2", 4, 300), ("id3", 5, 800)).toDF("id", "value", "cnt")

    df.printSchema()

 

    // 注册自定义UDF函数

    spark.udf.register("simpleUDF", (v: Int, w: Int) => v * v + w * w)

 

    df.select($"id", callUDF("simpleUDF", $"value", $"cnt")).toDF("id", "value").show

 

  }

}