寻找一种消除重复答案的方法

import hashlib

def deduplicate_exact(data_list):
    seen_hashes = set()
    unique_data = []
    
    for data in data_list:
        data_hash = hashlib.sha256(data.encode()).hexdigest()
        if data_hash not in seen_hashes:
            seen_hashes.add(data_hash)
            unique_data.append(data)
    
    return unique_data

# 示例使用
data_list = ["hello world", "hello world", "goodbye world"]
unique_data = deduplicate_exact(data_list)
print(unique_data)  # 输出: ['hello world', 'goodbye world']

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def deduplicate_fuzzy(data_list, threshold=0.8):
    vectorizer = TfidfVectorizer()
    tfidf_matrix = vectorizer.fit_transform(data_list)
    similarity_matrix = cosine_similarity(tfidf_matrix)
    
    unique_data = []
    seen_indices = set()
    
    for i in range(len(data_list)):
        if i not in seen_indices:
            similar_indices = similarity_matrix[i] > threshold
            similar_indices[i] = False  # 排除自身
            if similar_indices.any():
                seen_indices.update(similar_indices[similar_indices].index)
            unique_data.append(data_list[i])
    
    return unique_data

# 示例使用
data_list = ["hello world", "hello world!", "goodbye world"]
unique_data = deduplicate_fuzzy(data_list)
print(unique_data)  # 输出: ['hello world', 'goodbye world']