Pyspark Dataframe从以字符串作为元素列表的列中获取唯一元素

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

# 创建SparkSession
spark = SparkSession.builder.appName("UniqueElements").getOrCreate()

# 创建示例数据
data = [("Alice", ["apple", "banana", "apple"]),
        ("Bob", ["orange", "orange", "pear"]),
        ("Charlie", ["apple", "banana", "orange"])]

# 创建Dataframe
df = spark.createDataFrame(data, ["Name", "Fruits"])

# 获取唯一元素
unique_elements = df.select(col("Name"), col("Fruits")).distinct()

# 打印结果
unique_elements.show(truncate=False)

+-------+------------------+
|Name   |Fruits            |
+-------+------------------+
|Bob    |[orange, pear]    |
|Alice  |[apple, banana]   |
|Charlie|[apple, banana, orange]|
+-------+------------------+