如何使用NLTK从段落中删除重复的句子？

import nltk
from nltk.tokenize import sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def remove_duplicate_sentences(paragraph):
    # 将段落分割成句子
    sentences = sent_tokenize(paragraph)
    
    # 去除停用词和标点符号
    stop_words = set(stopwords.words("english"))
    
    # 初始化词形还原器
    lemmatizer = WordNetLemmatizer()
    
    # 对每个句子进行处理
    processed_sentences = []
    for sentence in sentences:
        # 将句子转换为小写并分词
        words = word_tokenize(sentence.lower())
        
        # 去除停用词和标点符号
        words = [word for word in words if word.isalnum() and word not in stop_words]
        
        # 词形还原
        words = [lemmatizer.lemmatize(word) for word in words]
        
        # 将处理后的句子重新组合
        processed_sentence = " ".join(words)
        processed_sentences.append(processed_sentence)
    
    # 使用TF-IDF向量化句子
    vectorizer = TfidfVectorizer()
    sentence_vectors = vectorizer.fit_transform(processed_sentences)
    
    # 计算句子之间的余弦相似度
    similarity_matrix = cosine_similarity(sentence_vectors)
    
    # 标记要删除的句子
    to_remove = set()
    for i in range(len(similarity_matrix)):
        for j in range(i+1, len(similarity_matrix)):
            if similarity_matrix[i][j] > 0.8:  # 设置相似度阈值
                to_remove.add(j)
    
    # 删除重复的句子
    unique_sentences = [sentences[i] for i in range(len(sentences)) if i not in to_remove]
    
    # 返回删除重复句子后的段落
    return " ".join(unique_sentences)

paragraph = "Your paragraph here."
processed_paragraph = remove_duplicate_sentences(paragraph)
print(processed_paragraph)