创建一个统计spark数据帧中重复次数的列

from pyspark.sql import SparkSession
from pyspark.sql.functions import count
from pyspark.sql.window import Window

spark = SparkSession.builder.appName("DuplicateCount").getOrCreate()

df = spark.read.csv("your_file.csv", header=True, inferSchema=True)

windowSpec = Window.partitionBy(df.columns).orderBy(df.columns)
df_with_duplicates = df.withColumn("duplicate_count", count("*").over(windowSpec) - 1)

df_with_duplicates.show()