在pyspark中处理具有多个记录类型的单个文件

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when
from pyspark.sql.types import StructType, StructField, StringType, IntegerType

spark = SparkSession.builder.appName("MultiRecordFileProcessing").getOrCreate()

schema = StructType([
    StructField("record_type", StringType(), True),
    StructField("data", StringType(), True)
])

df = spark.read.format("text").schema(schema).load("path/to/file.txt")

df_processed = df.withColumn("record_type", when(col("record_type") == "Type1", "Type1 Processing")
                            .when(col("record_type") == "Type2", "Type2 Processing")
                            .otherwise("Unknown Type"))

df_processed.show()