如何使用Spark有效地检查列中的所有值？

from pyspark.sql import SparkSession

# 创建SparkSession对象
spark = SparkSession.builder.getOrCreate()

# 从文件加载数据创建DataFrame
df = spark.read.csv("data.csv", header=True, inferSchema=True)

# 选择要检查的列
column_to_check = "column_name"
column_values = df.select(column_to_check).distinct().collect()

# 打印所有值
for value in column_values:
    print(value[0])

from pyspark.sql import SparkSession

# 创建SparkSession对象
spark = SparkSession.builder.getOrCreate()

# 从文件加载数据创建临时表
df = spark.read.csv("data.csv", header=True, inferSchema=True)
df.createOrReplaceTempView("table_name")

# 执行SQL查询并获取所有值
column_to_check = "column_name"
query = f"SELECT DISTINCT {column_to_check} FROM table_name"
column_values = spark.sql(query).collect()

# 打印所有值
for value in column_values:
    print(value[0])