带有函数的pySpark withColumn

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when

# 创建 SparkSession
spark = SparkSession.builder.appName("example").getOrCreate()

# 示例 DataFrame
data = [("Alice", 34), ("Bob", 28), ("Catherine", 31)]
columns = ["name", "age"]
df = spark.createDataFrame(data, columns)

# 使用 withColumn 添加新列
df_with_new_column = df.withColumn("is_old", when(col("age") > 30, True).otherwise(False))

df_with_new_column.show()

df = df.withColumn("is_old", when(col("age") > 30, True).otherwise(False))

from pyspark.sql.functions import lit

# 错误示例
df.withColumn("is_old", "True")  # 这会报错，因为 "True" 不是有效的表达式

# 正确示例
df.withColumn("is_old", lit(True))  # 使用 lit 函数来创建常量列