从嵌套数组和结构spark中提取值

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

spark = SparkSession.builder.getOrCreate()

data = [
    (1, [2, 3, 4], ("a", "b")),
    (2, [5, 6], ("c", "d")),
    (3, [7], ("e", "f"))
]

df = spark.createDataFrame(data, ["id", "array_col", "struct_col"])

# 使用getItem函数提取指定位置的元素
df.select(col("array_col").getItem(0).alias("array_element_1"),
          col("array_col").getItem(1).alias("array_element_2")).show()

+---------------+---------------+
|array_element_1|array_element_2|
+---------------+---------------+
|              2|              3|
|              5|              6|
|              7|           null|
+---------------+---------------+

# 使用点操作符提取结构中的字段
df.select(col("struct_col._1").alias("struct_field_1"),
          col("struct_col._2").alias("struct_field_2")).show()

+--------------+--------------+
|struct_field_1|struct_field_2|
+--------------+--------------+
|             a|             b|
|             c|             d|
|             e|             f|
+--------------+--------------+