将具有数组形状的字符串的pyspark databricks数据框转换为标准列

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, explode, split

spark = SparkSession.builder.getOrCreate()

# 读取数据为数据框
data = spark.createDataFrame([(1, "a,b,c"), (2, "d,e,f,g"), (3, "h,i")], ["id", "values"])
data.show()

+---+-------+
| id| values|
+---+-------+
|  1|  a,b,c|
|  2|d,e,f,g|
|  3|    h,i|
+---+-------+

# 将values列按逗号分割成数组
data = data.withColumn("array_values", split(col("values"), ","))

# 将数组展开为多行
data = data.withColumn("value", explode(col("array_values")))
data.show()

+---+-------+------------+-----+
| id| values|array_values|value|
+---+-------+------------+-----+
|  1|  a,b,c|  [a, b, c]|    a|
|  1|  a,b,c|  [a, b, c]|    b|
|  1|  a,b,c|  [a, b, c]|    c|
|  2|d,e,f,g| [d, e, f, g]|    d|
|  2|d,e,f,g| [d, e, f, g]|    e|
|  2|d,e,f,g| [d, e, f, g]|    f|
|  2|d,e,f,g| [d, e, f, g]|    g|
|  3|    h,i|      [h, i]|    h|
|  3|    h,i|      [h, i]|    i|
+---+-------+------------+-----+

# 将展开后的数组元素作为单独的列
data = data.groupBy("id").pivot("value").count().na.fill(0)
data.show()

+---+---+---+---+---+---+---+---+
| id|  a|  b|  c|  d|  e|  f|  g|
+---+---+---+---+---+---+---+---+
|  1|  1|  1|  1|  0|  0|  0|  0|
|  2|  0|  0|  0|  1|  1|  1|  1|
|  3|  0|  0|  0|  0|  0|  0|  0|
+---+---+---+---+---+---+---+---+