如何基于Pandas中行之间的文本相似度来消除重复

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

data = {'text': ['This is the first sentence.',
                 'This is the second sentence.',
                 'This is the third sentence.',
                 'This is the fourth sentence.',
                 'This is the first sentence.']}
df = pd.DataFrame(data)

vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform(df['text'])

similarity_matrix = cosine_similarity(vectors)

duplicates = []
for i in range(len(similarity_matrix)):
    for j in range(i+1, len(similarity_matrix)):
        if similarity_matrix[i][j] == 1:
            duplicates.append(j)

df.drop(duplicates, inplace=True)
df.reset_index(drop=True, inplace=True)