解析Python Spark中表列中存储的JSON

from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json
from pyspark.sql.types import StructType, StructField, StringType

spark = SparkSession.builder.appName("JSON Parsing").getOrCreate()

# 定义表结构
schema = StructType([
    StructField("name", StringType(), True),
    StructField("age", StringType(), True),
    StructField("city", StringType(), True)
])

# 定义JSON模式
json_schema = schema.json()

df = spark.read.format("json").load("path/to/json/file")

parsed_df = df.withColumn("parsed_data", from_json(df["json_column"], json_schema))

parsed_df.select("parsed_data.name", "parsed_data.age").show()

parsed_df.filter(parsed_df["parsed_data.age"] > 18).show()

parsed_df.groupBy("parsed_data.city").count().show()