选择distinct值pandas中的groupby列

import pandas as pd

# 创建示例DataFrame
data = {
    'ProductCategory': ['A', 'A', 'B', 'B', 'A'],
    'ProductName': ['Product1', 'Product2', 'Product3', 'Product3', 'Product4']
}
df = pd.DataFrame(data)

# 使用groupby获取每个产品类别中的不同产品名称
distinct_products = df.groupby('ProductCategory')['ProductName'].unique()

print(distinct_products)

# 使用分块处理和set来提高效率
distinct_products_chunked = []

for chunk in pd.read_csv('large_data.csv', chunksize=1000):
    distinct_products_chunked.append(chunk.groupby('ProductCategory')['ProductName'].apply(set))

# 合并结果
final_distinct_products = {}
for chunk in distinct_products_chunked:
    for category, products in chunk.items():
        if category not in final_distinct_products:
            final_distinct_products[category] = set(products)
        else:
            final_distinct_products[category].update(products)

print(final_distinct_products)