使用Spark Scala从YAML文件中提取键、值对

import org.apache.spark.sql.SparkSession
import org.yaml.snakeyaml.Yaml

val spark = SparkSession.builder()
  .appName("YAML Parsing")
  .master("local[*]") // 设置本地模式，*表示使用所有可用的线程
  .getOrCreate()

val yamlFile = "path/to/yaml/file.yaml"
val yamlContent = spark.sparkContext.wholeTextFiles(yamlFile).map(_._2).collect.mkString("\n")

val yaml = new Yaml()
val yamlData = yaml.load(yamlContent)

val keyValues = scala.collection.mutable.Map[String, Any]()
def extractKeyValues(data: Any, prefix: String = ""): Unit = {
  data match {
    case map: java.util.LinkedHashMap[String, Any] =>
      map.forEach((key, value) => extractKeyValues(value, s"$prefix$key."))
    case list: java.util.ArrayList[Any] =>
      list.forEach(value => extractKeyValues(value, prefix))
    case _ =>
      keyValues += (prefix -> data)
  }
}
extractKeyValues(yamlData)

keyValues.foreach { case (key, value) =>
  println(s"$key: $value")
}