将字典中的值解析为pyspark中的列表

from pyspark.sql import SparkSession
from pyspark.sql.functions import explode

# 创建SparkSession
spark = SparkSession.builder.appName("Dictionary to List").getOrCreate()

# 示例数据
data = [
    {"id": 1, "values": [10, 20, 30]},
    {"id": 2, "values": [40, 50]},
    {"id": 3, "values": [60, 70, 80, 90]}
]

# 创建DataFrame
df = spark.createDataFrame(data)

# 将字典中的值解析为列表
df_exploded = df.select("id", explode("values").alias("value"))

# 显示结果
df_exploded.show()

from pyspark.sql.functions import col, when, array

# 添加类型检查
df = df.withColumn("values", when(col("values").isNotNull() & (col("values").cast("string").contains("[")), col("values")).otherwise(array()))

# 过滤掉空行
df_exploded = df_exploded.filter(col("value").isNotNull())