删除Python中某列的相似文本行

import pandas as pd
import difflib

df = pd.read_csv('data.csv')

def similarity(text1, text2):
    seq = difflib.SequenceMatcher(None, text1, text2)
    return seq.ratio()

column_name = 'column_name'  # 指定要删除相似文本的列名
threshold = 0.8  # 设定相似度的阈值

for i, row in df.iterrows():
    current_text = row[column_name]
    for j, compare_row in df.iterrows():
        if i != j:
            compare_text = compare_row[column_name]
            if similarity(current_text, compare_text) > threshold:
                df.drop(j, inplace=True)

df.to_csv('processed_data.csv', index=False)