查找从一个csv列到其他csv列的单词的频率

import pandas as pd

df = pd.read_csv('file1.csv', usecols=['target_column'])

df.dropna(inplace=True)  # 删除空值
df.drop_duplicates(inplace=True)  # 删除重复项

import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer

nltk.download('punkt')
nltk.download('stopwords')

stop_words = set(stopwords.words('english'))
ps = PorterStemmer()

def process_text(text):
    tokens = word_tokenize(text.lower())  # 分词并转换为小写
    tokens = [token for token in tokens if token.isalpha()]  # 仅保留字母字符
    tokens = [token for token in tokens if token not in stop_words]  # 去除停用词
    tokens = [ps.stem(token) for token in tokens]  # 词干提取
    return tokens

df['processed_text'] = df['target_column'].apply(process_text)

from nltk import FreqDist

word_freq = FreqDist([word for text in df['processed_text'] for word in text])

word_freq_df = pd.DataFrame.from_dict(word_freq, orient='index', columns=['frequency'])
word_freq_df.index.name = 'word'
word_freq_df.sort_values(by='frequency', ascending=False, inplace=True)

word_freq_df.to_csv('word_frequency.csv')