PySpark分解json字符串

from pyspark.sql import SparkSession
from pyspark.sql.functions import from_json
from pyspark.sql.types import StructType, StructField, StringType

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据
data = [
    ('{"name": "John", "age": 30}',),
    ('{"name": "Alice", "age": 25}',),
    ('{"name": "Bob", "age": 35}',)
]
df = spark.createDataFrame(data, ['json_string'])

# 定义JSON模式
schema = StructType([
    StructField('name', StringType()),
    StructField('age', StringType())
])

# 使用from_json函数解析JSON字符串
df = df.withColumn('json_object', from_json(df.json_string, schema))

# 显示结果
df.show(truncate=False)

+-----------------------+----------------+
|json_string            |json_object     |
+-----------------------+----------------+
|{"name": "John", "age": 30}|[John, 30]      |
|{"name": "Alice", "age": 25}|[Alice, 25]     |
|{"name": "Bob", "age": 35}  |[Bob, 35]       |
+-----------------------+----------------+