pySpark DataFrame:如何并行比较两个数据帧的列？

from pyspark.sql import SparkSession
from pyspark.sql.functions import col
from pyspark.sql.types import BooleanType

spark = SparkSession.builder.appName("DataFrameComparison").getOrCreate()

df1 = spark.read.csv("dataframe1.csv", header=True, inferSchema=True)
df2 = spark.read.csv("dataframe2.csv", header=True, inferSchema=True)

def compare_columns(col1, col2):
    # 在这里实现你的比较逻辑
    if col1 == col2:
        return True
    else:
        return False

compare_udf = udf(compare_columns, BooleanType())

result_df = df1.join(df2, on="key_column").withColumn("column_comparison", compare_udf(col("df1_column"), col("df2_column")))