连接65,000个.csv文件的高效方法

import pandas as pd
from concurrent.futures import ProcessPoolExecutor

def process_chunk(chunk):
    # 这里可以添加数据处理逻辑
    return chunk.sum()  # 示例：计算每块的和

def merge_csv_files(file_paths, output_path, chunk_size=10000):
    all_results = []
    
    with ProcessPoolExecutor() as executor:
        futures = []
        for file_path in file_paths:
            reader = pd.read_csv(file_path, chunksize=chunk_size)
            for chunk in reader:
                futures.append(executor.submit(process_chunk, chunk))
        
        for future in futures:
            all_results.append(future.result())
    
    final_result = pd.concat(all_results)
    final_result.to_csv(output_path, index=False)

# 使用示例
file_paths = ['file1.csv', 'file2.csv', ..., 'file65000.csv']  # 替换为实际文件路径
merge_csv_files(file_paths, 'merged_output.csv')