Spark 'join‘DataFrame with List and return String

import org.apache.spark.sql.SparkSession

// 创建SparkSession
val spark = SparkSession.builder()
  .appName("Join DataFrame with List")
  .master("local")
  .getOrCreate()

// 创建主DataFrame
val mainDF = spark.createDataFrame(Seq(
  (1, "Alice"),
  (2, "Bob"),
  (3, "Charlie")
)).toDF("id", "name")

// 创建次DataFrame
val list = List((1, "Engineer"), (2, "Doctor"), (3, "Teacher"))
val listDF = spark.createDataFrame(list).toDF("id", "profession")

// 将次DataFrame转换为List
val professionList = listDF.select("profession").rdd.map(r => r(0).asInstanceOf[String]).collect().toList

// 进行join操作
val resultDF = mainDF.join(listDF, Seq("id"))

// 对结果进行处理
val resultString = resultDF.select("name", "profession").orderBy("id").collect().map(row => row.mkString(", ")).mkString("; ")

// 输出结果
println(resultString)