Pyspark -将时间戳传递给udf

from pyspark.sql import SparkSession
from pyspark.sql.functions import udf
from pyspark.sql.types import TimestampType

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例数据集
data = [(1, 1612345678), (2, 1612345679), (3, 1612345680)]
df = spark.createDataFrame(data, ["id", "timestamp"])

# 定义UDF来处理时间戳
def process_timestamp(timestamp):
    # 将时间戳转换为日期时间格式
    datetime = spark.sql("SELECT from_unixtime({})".format(timestamp)).collect()[0][0]
    # 进行进一步的处理，例如提取日期、时间等
    # ...

    return datetime

# 注册UDF
process_timestamp_udf = udf(process_timestamp, TimestampType())

# 使用UDF处理时间戳
df = df.withColumn("datetime", process_timestamp_udf(df["timestamp"]))

# 显示结果
df.show()