高效地迭代字符串列表以获得成对的WMD距离矩阵

pip install gensim numpy

import numpy as np
from gensim.models import Word2Vec
from gensim.models.keyedvectors import KeyedVectors
from nltk.tokenize import word_tokenize
from scipy.spatial.distance import cosine

# 假设你已经有了一个预训练的Word2Vec模型或者词向量文件
# model = Word2Vec.load("your_pretrained_model")
# 或者加载词向量文件
model = KeyedVectors.load_word2vec_format('path_to_word2vec_file', binary=True)

# 示例字符串列表
string_list = [
    "This is a sample sentence.",
    "Another example sentence.",
    "Yet another sentence for testing."
]

def preprocess(text):
    return [word for word in word_tokenize(text.lower()) if word in model.wv]

def wmd_distance(s1, s2):
    s1 = preprocess(s1)
    s2 = preprocess(s2)
    distance = model.wmdistance(s1, s2)
    return distance

n = len(string_list)
distance_matrix = np.zeros((n, n))

for i in range(n):
    for j in range(i + 1, n):
        distance_matrix[i, j] = wmd_distance(string_list[i], string_list[j])
        distance_matrix[j, i] = distance_matrix[i, j]  # WMD距离是对称的

print(distance_matrix)