2021年大数据Spark（三十）：SparkSQL自定义UDF函数

package cn.itcast.sql

import org.apache.spark.SparkContext
import org.apache.spark.sql.expressions.UserDefinedFunction
import org.apache.spark.sql.{DataFrame, SparkSession}

/**
 * Author itcast
 * Desc
 * 将udf.txt中的单词使用SparkSQL自定义函数转为大写
 * hello
 * haha
 * hehe
 * xixi
 */
object UDFDemo {
  def main(args: Array[String]): Unit = {
    //1.准备环境
    val spark: SparkSession = SparkSession.builder().appName("UDFDemo").master("local[*]").getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    import spark.implicits._

    //2.加载数据
    val df: DataFrame = spark.read.text("data/input/udf.txt")
    df.show()
    /*
    +-----+
    |value|
    +-----+
    |hello|
    | haha|
    | hehe|
    | xixi|
    +-----+
     */

    //3.使用自定义函数将单词转为大写

    //SQL风格-自定义函数
    //spark.udf.register("函数名",函数实现)
    spark.udf.register("small2big", (value: String) => value.toUpperCase())

    df.createOrReplaceTempView("t_words")
    val sql: String =
      """
        |select value,small2big(value) as small_word
        |from t_words
        |""".stripMargin
    spark.sql(sql).show() //如果没有自定义该函数,那么会报错:Undefined function: 'small2big'.

    //DSL风格-自定义函数
    //val small2big2: UserDefinedFunction = functions.udf((value: String) => value.toUpperCase)
    import org.apache.spark.sql.functions._
    val small2big2: UserDefinedFunction = udf((value: String) => value.toUpperCase)
    df.select($"value", small2big2($"value").as("small_word")).show()
  }
}