筛选数组大小=1 pyspark的行出错

from pyspark.sql import SparkSession
from pyspark.sql.functions import size, col

# 创建 SparkSession
spark = SparkSession.builder.appName("example").getOrCreate()

# 示例数据
data = [
    (1, [1]),
    (2, [1, 2]),
    (3, [3])
]

# 创建 DataFrame
columns = ["id", "values"]
df = spark.createDataFrame(data, columns)

# 筛选数组大小为 1 的行
filtered_df = df.filter(size(col("values")) == 1)

filtered_df.show()

# 过滤掉空数组
filtered_df = df.filter(size(col("values")) > 0).filter(size(col("values")) == 1)

filtered_df.show()

from pyspark.sql.types import ArrayType, IntegerType

# 定义 schema
schema = StructType([
    StructField("id", IntegerType(), True),
    StructField("values", ArrayType(IntegerType()), True)
])

# 创建 DataFrame
df = spark.createDataFrame(data, schema)

# 筛选数组大小为 1 的行
filtered_df = df.filter(size(col("values")) == 1)

filtered_df.show()