使用中文维基百科语料库训练一个word2vec模型并使用说明

python WikiExtractor.py -b 500M -o zhwiki zhwiki-20190401-pages-articles-multistream.xml.bz2

import logging,jieba,os,re

def get_stopwords():
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)
    #加载停用词表
    stopword_set = set()
    with open("../stop_words/stopwords.txt",'r',encoding="utf-8") as stopwords:
        for stopword in stopwords:
            stopword_set.add(stopword.strip("\n"))
    return stopword_set

'''
使用正则表达式解析文本
'''
def parse_zhwiki(read_file_path,save_file_path):
    #过滤掉<doc>
    regex_str = "[^<doc.*>$]|[^</doc>$]"
    file = open(read_file_path,"r",encoding="utf-8")
    #写文件
    output = open(save_file_path,"w+",encoding="utf-8")
    content_line = file.readline()
    #获取停用词表
    stopwords = get_stopwords()
     #定义一个字符串变量，表示一篇文章的分词结果
    article_contents = ""
    while content_line:
        match_obj = re.match(regex_str,content_line)
        content_line = content_line.strip("\n")
        if len(content_line) > 0:
            if match_obj:
                #使用jieba进行分词
                words = jieba.cut(content_line,cut_all=False)
                for word in words:
                    if word not in stopwords:
                        article_contents += word+" "
            else:
                if len(article_contents) > 0:
                    output.write(article_contents+"\n")
                    article_contents = ""
        content_line = file.readline()
    output.close()

'''
合并分词后的文件
'''
def merge_corpus():
    output = open("../dataset/zhwiki/BB/wiki_corpus","w",encoding="utf-8")
    input = "../dataset/zhwiki/BB"
    for i in range(3):
        file_path = os.path.join(input,str("wiki_corpus0%s"%str(i)))
        file = open(file_path,"r",encoding="utf-8")
        line = file.readline()
        while line:
            output.writelines(line)
            line = file.readline()
        file.close()
    output.close()

import logging
from gensim.models import word2vec

def main():
    logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s",level=logging.INFO)
    sentences = word2vec.LineSentence("../dataset/zhwiki/BB/wiki_corpus")
    # size：单词向量的维度。
    model = word2vec.Word2Vec(sentences,size=250)
    #保存模型
    model.save("../model/wiki_corpus.bin")
    # model.save("../model/wiki_corpus.model")
    # model.wv.save_word2vec_format("./sogou_word2vec/min_count-1/sogou.wor2vec.txt")

if __name__ == "__main__":
    main()

logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s",level=logging.INFO)
    model = models.Word2Vec.load("../wiki_chinese/model/wiki_corpus.bin")
    #输入一个词找出相似的前10个词
    one_corpus = ["人工智能"]

    # ''' 词库中与one_corpus最相似的10个词'''
    result = model.most_similar(one_corpus[0],topn=10)
    print(result)

# 两个词的相似度
# #输入两个词计算相似度
two_corpus = ["腾讯","阿里巴巴"]
res = model.similarity(two_corpus[0],two_corpus[1])
print("similarity:%.4f"%res)