概率论：从赌场算法到菲尔兹奖的奇妙旅程

import jieba
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.naive_bayes import MultinomialNB

def seg_sentence(sentence, stopwords_path):
    """
    对句子进行分词
    """
    print("now token sentence...")
    
    def stopwordslist(filepath):
        """
        创建停用词list闭包
        """
        # 注意：此处原文件为 'rb' 读取并 decode('utf-8')，实际运行时需注意路径和编码
        stopwords = [line.decode('utf-8').strip() for line in open(filepath, 'rb').readlines()]
        return stopwords

    sentence_seged = jieba.cut(sentence.strip())
    stopwords = stopwordslist('C:/Users/KL/Desktop/stopwords.txt')  # 停用词路径
    outstr = ''
    for word in sentence_seged:
        if word not in stopwords and word != '\t':
            outstr += word
    return outstr + " "

def tokenFile(file_path_Predict, write_path_Predict):
    """
    对文本进行分词，结果存储在write_path
    """
    with open(write_path_Predict, 'w', encoding='utf-8') as w:
        with open(file_path_Predict, 'r', encoding='utf-8') as f:
            for line in f.readlines():
                line = line.strip()
                # 假设文件格式为：标签\t文本内容
                token_sen = seg_sentence(line.split('\t')[1], 'stopwords.txt')
                w.write(line.split('\t')[0] + "\t" + token_sen + "\n")

def tokenFilePredict(file_path, write_path):
    # 对待预测的文本文件进行分词
    with open(write_path, 'w', encoding='utf-8') as w:
        with open(file_path, 'r', encoding='utf-8') as f:
            for line in f.readlines():
                line = line.strip()
                # 假设待预测文件格式为：文本内容
                token_sen = seg_sentence(line.split('\t')[0], 'stopwords.txt')
                w.write(token_sen + "\n")

def constructDataset(path):
    """
    构建数据集: 返回标签列表和语料列表
    """
    label_list = []
    corpus_list = []
    with open(path, 'r', encoding='utf-8') as p:
        for line in p.readlines():
            label_list.append(line.split('\t')[0])
            corpus_list.append(line.split('\t')[1])
    return label_list, corpus_list

def constructDatasetPredict(path):
    # 构建待预测数据集
    corpus_list = []
    with open(path, 'r', encoding='utf-8') as p:
        for line in p.readlines():
            corpus_list.append(line.split('\t')[0])
    return corpus_list

if __name__ == '__main__':
    # 1. 文件路径配置
    file_path = 'C:/Users/KL/Desktop/dataSet.txt' 
    write_path = 'C:/Users/KL/Desktop/dataSet_token.txt' 
    file_path_Predict = 'C:/Users/KL/Desktop/dataSetPredict.txt' 
    write_path_Predict = 'C:/Users/KL/Desktop/dataSet_tokenPredict.txt' 
    write_path_Result = 'C:/Users/KL/Desktop/dataSet_Result.txt' 

    # 2. 执行分词
    tokenFile(file_path, write_path)
    tokenFilePredict(file_path_Predict, write_path_Predict)

    # 3. 构建数据集
    label, data = constructDataset(write_path)
    data1 = constructDatasetPredict(write_path_Predict)

    # 4. 划分训练测试集
    x_train, x_test, y_train, y_test = train_test_split(data, label,
                                                        test_size=0.25,
                                                        random_state=42)

    # 5. 特征工程 (使用 TF-IDF 提取特征)
    tfidf_vect = TfidfVectorizer(analyzer='word', max_features=5000)
    tfidf_vect.fit(x_train)
    xtrain_tfidf = tfidf_vect.transform(x_train)
    xtest_tfidf = tfidf_vect.transform(x_test)
    x_tfidf = tfidf_vect.transform(data1)

    # 6. 模型训练与预测 (多项式朴素贝叶斯)
    mnb_count = MultinomialNB()
    mnb_count.fit(xtrain_tfidf, y_train)
    print(mnb_count.score(xtest_tfidf, y_test)) # 打印模型在测试集上的准确率
    y_predict = mnb_count.predict(x_tfidf) # 对新数据进行预测

    # 7. 结果输出
    with open(write_path_Result, 'w', encoding='utf-8') as w:
        # 假设预测数据总数为 1500 条
        for i in range(0, 1499):
            w.write(y_predict[i] + "\n")