[文本语义相似] 基于ngram-tf-idf的余弦距离(gensim实现)

import os
import jieba
import pickle
import logging
import numpy as np
from gensim import corpora, models, similarities
 
 
class StopWords(object):
    '''
    '''
    def __init__(self, stopwords_file=stopwords_file ):
        self.stopwords = set( [ word.strip() for word in open(stopwords_file, 'r') ] )
    
    def del_stopwords(self, words):
        return [ word for word in words if word not in self.stopwords ]
 
stop_word = StopWords()

# gen 3-gram
def _list_3_ngram(words, n=3, m=2):
    pattern1 = re.compile(r'[0-9]')
    if len(words) < n:
        n = len(words)
    temp=[words[i - k:i] for k in range(m, n + 1) for i in range(k, len(words) + 1) ]
    return [item for item in temp if len(''.join(item).strip())>0 and len(pa