在Scala中查找数据帧中数组列的重复值

import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.functions._

val spark = SparkSession.builder()
  .appName("Duplicate Values")
  .master("local")
  .getOrCreate()

val data = Seq(
  (1, Array(1, 2, 3)),
  (2, Array(4, 5, 6)),
  (3, Array(1, 2, 3))
)

val df = spark.createDataFrame(data).toDF("id", "arrayCol")

val duplicates = df.groupBy("arrayCol")
  .agg(count("id").alias("count"))
  .filter(col("count") > 1)

duplicates.show(false)