在pyspark中高效地添加大量数据帧

from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 定义模式
schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", IntegerType(), True),
    # 添加其他字段...
])

# 创建空的数据帧
df = spark.createDataFrame([], schema)

data = [("Alice", 25), ("Bob", 30), ("Charlie", 35)]

# 将数据添加到数据帧
df = df.unionAll(spark.createDataFrame(data, schema))

def add_data_to_dataframe(iterator):
    # 创建SparkSession
    spark = SparkSession.builder.getOrCreate()

    # 定义模式
    schema = StructType([
        StructField("name", StringType(), True),
        StructField("age", IntegerType(), True),
        # 添加其他字段...
    ])

    # 创建空的数据帧
    df = spark.createDataFrame([], schema)

    for row in iterator:
        # 将数据添加到数据帧
        df = df.unionAll(spark.createDataFrame([row], schema))

    return df

# 准备要添加的数据
data = [("Alice", 25), ("Bob", 30), ("Charlie", 35)]
# 将数据分成多个分区
data_partitions = [data[i:i+100] for i in range(0, len(data), 100)]

# 将数据分批添加到数据帧
df = spark.sparkContext.parallelize(data_partitions).mapPartitions(add_data_to_dataframe).reduce(lambda df1, df2: df1.unionAll(df2))