统计多个csv文件中不带停用词的词频

import csv
import re
from collections import Counter

# 定义停用词列表
stopwords = ['的', '是', '在', '了', '和', '等']

# 合并多个CSV文件为一个数据集
file_paths = ['file1.csv', 'file2.csv', 'file3.csv']
data = []
for file_path in file_paths:
    with open(file_path, 'r', encoding='utf-8') as file:
        reader = csv.reader(file)
        for row in reader:
            data.extend(row)

# 数据清洗和分词处理
words = []
for text in data:
    # 去除标点符号和特殊字符
    text = re.sub(r'[^\w\s]', '', text)
    # 转换为小写字母
    text = text.lower()
    # 分词处理
    words.extend(text.split())

# 停用词过滤
words = [word for word in words if word not in stopwords]

# 词频统计
word_freq = Counter(words)

# 按词频降序排列
sorted_word_freq = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)

# 输出结果
for word, freq in sorted_word_freq:
    print(word, freq)