基于可用值的多列Scala Spark数据帧过滤器

import org.apache.spark.sql.SparkSession

// 创建SparkSession
val spark = SparkSession.builder()
  .appName("DataFrameFilter")
  .master("local")
  .getOrCreate()

// 创建示例数据帧
val data = Seq(
  ("Alice", 25, "New York"),
  ("Bob", 30, null),
  ("Charlie", null, "San Francisco"),
  (null, null, null)
)
val columns = Seq("name", "age", "city")
val df = spark.createDataFrame(data).toDF(columns: _*)

// 定义过滤条件函数
val filterFunc = (row: org.apache.spark.sql.Row) => {
  val name = row.getAs[String]("name")
  val age = row.getAs[Integer]("age")
  val city = row.getAs[String]("city")
  
  // 过滤条件：age不为空且city不为空
  age != null && city != null
}

// 应用过滤器
val filteredDF = df.filter(filterFunc)

// 显示过滤后的结果
filteredDF.show()