Avro schema ( .avsc )在Pyspark中的实施

from pyspark.sql import SparkSession
from pyspark.sql.functions import col
import avro.schema
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter

spark = SparkSession.builder.appName("AvroExample").getOrCreate()

# 读取Avro文件
df = spark.read.format("avro").load("path/to/avro/file.avro")

# 显示数据
df.show()

# 创建示例数据
data = [
    {"name": "John", "age": 30},
    {"name": "Alice", "age": 25}
]

# 创建Avro schema
schema = avro.schema.parse('''
    {
        "type": "record",
        "name": "Person",
        "fields": [
            {"name": "name", "type": "string"},
            {"name": "age", "type": "int"}
        ]
    }
''')

# 写入Avro文件
with open("path/to/output/file.avro", "wb") as f:
    writer = DataFileWriter(f, DatumWriter(), schema)
    for record in data:
        writer.append(record)
    writer.close()