Spark中两个结构相同的数据帧的比较

import org.apache.spark.sql.SparkSession

val spark = SparkSession.builder()
  .appName("DataFrameComparison")
  .getOrCreate()

import spark.implicits._

// 创建两个相同结构的数据帧
val df1 = Seq(
  (1, "Alice"),
  (2, "Bob"),
  (3, "Charlie")
).toDF("id", "name")

val df2 = Seq(
  (1, "Alice"),
  (2, "Bob"),
  (3, "Charlie")
).toDF("id", "name")

// 使用equals方法比较两个数据帧
val isEqual = df1.equals(df2)
println(s"Are the dataframes equal? $isEqual")

// 使用except方法获取在df1中存在而在df2中不存在的行
val diff1 = df1.except(df2)
println("Rows in df1 but not in df2:")
diff1.show()

// 使用subtract方法获取在df1中存在而在df2中不存在的行
val diff2 = df1.subtract(df2)
println("Rows in df1 but not in df2:")
diff2.show()

// 使用join方法比较两个数据帧
val joinResult = df1.join(df2, Seq("id"), "left_anti")
println("Join result:")
joinResult.show()