在使用pyspark和预定义的结构模式读取嵌套JSON时，如何将缺少的列添加为null

from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json
from pyspark.sql.types import StructType, StructField, StringType

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 定义预期的结构模式
schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", StringType(), True),
    StructField("gender", StringType(), True),
    StructField("address", StringType(), True)
])

# 读取嵌套JSON数据
json_data = [
    '{"name": "Alice", "age": "25", "gender": "Female"}',
    '{"name": "Bob", "age": "30"}',
    '{"name": "Charlie", "gender": "Male"}'
]
df = spark.read.json(spark.sparkContext.parallelize(json_data))

# 将缺少的列添加为null值
df = df.withColumn("data", from_json(df.value, schema)).select("data.*")

# 显示DataFrame
df.show()