Python / Pyspark - Count NULL、empty和NaN

import pandas as pd
import numpy as np

# 创建一个示例数据集
data = pd.DataFrame({'A': [1, 2, np.nan, 4],
                     'B': [np.nan, '', 'abc', 'def'],
                     'C': [np.nan, np.nan, np.nan, np.nan]})

# 计算NULL值的数量
null_count = data.isnull().sum().sum()

# 计算empty值的数量
empty_count = (data == '').sum().sum()

# 计算NaN值的数量
nan_count = data.isna().sum().sum()

print("NULL值的数量：", null_count)
print("empty值的数量：", empty_count)
print("NaN值的数量：", nan_count)

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, isnan, isnull

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建一个示例数据集
data = spark.createDataFrame([(1, None, None), (2, '', None), (None, 'abc', None), (4, 'def', None)], ['A', 'B', 'C'])

# 计算NULL值的数量
null_count = data.select([col(c).isNull().cast('int').alias(c) for c in data.columns]).agg(*[sum(c).alias(c) for c in data.columns]).collect()[0].asDict()

# 计算empty值的数量
empty_count = data.select([col(c).isNull().cast('int').alias(c) for c in data.columns]).filter((col('B') == '') | (col('C') == '')).agg(*[sum(c).alias(c) for c in data.columns]).collect()[0].asDict()

# 计算NaN值的数量
nan_count = data.select([isnan(c).cast('int').alias(c) for c in data.columns]).agg(*[sum(c).alias(c) for c in data.columns]).collect()[0].asDict()

print("NULL值的数量：", null_count)
print("empty值的数量：", empty_count)
print("NaN值的数量：", nan_count)