PyTorch自然语言处理入门与实战

#!/usr/bin/env python3
import sys
import os
import time

class TextSpliter(object):
    def __init__(self, corpus_path, encoding='utf8', max_load_word_length=4):
        self.dict = {}
        self.dict2 = {}
        self.max_word_length = 1
        begin_time = time.time()
        print('start load corpus from %s' % corpus_path)
        # 加载语料
 with open(corpus_path, 'r', encoding=encoding) as f:
            for l in f:
                l.replace('[', '')
                l.replace(']', '')
                wds = l.strip().split('  ')

                last_wd = ''
                for i in range(1, len(wds)): # 下标从1开始，因为每行第一个词是标签
                    try:
                        wd, wtype = wds[i].split('/')
                    except:
                        continue
                    if len(wd) == 0 or len(wd) > max_load_word_length or not wd.isalpha():
                        continue
                    if wd not in self.dict:
                        self.dict[wd] = 0
                        if len(wd) > self.max_word_length:
                            # 更新最大词长度
                            self.max_word_length = len(wd)
                            print('max_word_length=%d, word is %s' %(self.max_word_length, wd))
                    self.dict[wd] += 1
                    if last_wd:
                        if last_wd+':'+wd not in self.dict2:
                            self.dict2[last_wd+':'+wd] = 0
                        self.dict2[last_wd+':'+wd] += 1
                    last_wd = wd
        self.words_cnt = 0
        max_c = 0
        for wd in self.dict:
            self.words_cnt += self.dict[wd]
            if self.dict[wd] > max_c:
                max_c = self.dict[wd]
        self.words2_cnt = sum(self.dict2.values())
        print('load corpus finished, %d words in dict and frequency is %d, %d words in dict2 frequency is %d' % (len(self.dict),len(self.dict2), self.words_cnt, self.words2_cnt), 'msg')
        print('%f seconds elapsed' % (time.time()-begin_time), 'msg')

    def split(self, text):
        sentence = ''
        result = ''
        for ch in text:
            if not ch.isalpha():
                result += self.__split_sentence__(sentence) + ' ' + ch + ' '
                sentence = ''
            else:
                sentence += ch
        return result.strip(' ')

    def __get_a_split__(self, cur_split, i):
        if i >= len(self.cur_sentence):
            self.split_set.append(cur_split)
            return
  j = min(self.max_word_length, len(self.cur_sentence) - i + 1)
        while j > 0:
            if j == 1 or self.cur_sentence[i:i+j] in self.dict:
                self.__get_a_split__(cur_split + [self.cur_sentence[i:i+j]], i+j)
                if j == 2:
                    break
            j -= 1

def __get_cnt__(self, dictx, key):
    # 获取出现次数
        try:
            return dictx[key] + 1
        except KeyError:
            return 1

    def __get_word_probablity__(self, wd, pioneer=''):
        if pioneer == '':
            return self.__get_cnt__(self.dict, wd) / self.words_cnt
        return self.__get_cnt__(self.dict2, pioneer + ':' + wd) / self.__get_cnt__(self.dict, pioneer)

    def __calc_probability__(self, sequence):
        probability = 1
        pioneer = ''
        for wd in sequence:
            probability *= self.__get_word_probablity__(wd, pioneer)
            pioneer = wd
        return probability

    def __split_sentence__(self, sentence):
        if len(sentence) == 0:
            return ''
        self.cur_sentence = sentence.strip()
        self.split_set = []
        self.__get_a_split__([], 0)
        print(sentence + str(len(self.split_set)))
        max_probability = 0
        for splitx in self.split_set:
            probability = self.__calc_probability__(splitx)  # 计算概率
            print(str(splitx)+ ' - ' +str(probability))
            if probability > max_probability:  # 测试是否超过当前记录的最高概率
                max_probability = probability
                best_split = splitx
        return ' '.join(best_split)  # 把list 拼接为字符串

if __name__ == '__main__':
    btime = time.time()
    base_path = os.path.dirname(os.path.realpath(__file__))
    spliter = TextSpliter(os.path.join(base_path, '199801.txt'))
    with open(os.path.join(base_path, 'test.txt'), 'r', encoding='utf8') as f:
        with open(os.path.join(base_path, 'result.txt',), 'w', encoding='utf8') as fr:
            for l in f:
                fr.write(spliter.split(l))
    print ('time elapsed %f' % (time.time() - btime))

并广泛动员社会各方面的力量
今天我们参观台塑钢铁厂的车间
今天天气很好
我这会先去吃个饭

import jieba
print('/'.join(list(jieba.cut("并广泛动员社会各方面的力量"))))

Building prefix dict from the default dictionary ...
Dumping model to file cache C:\Users\sxwxs\AppData\Local\Temp\jieba.cache
Loading model cost 0.969 seconds.
Prefix dict has been built successfully.
'并/广泛/动员/社会/各/方面/的/力量'

import pkuseg
seg = pkuseg.pkuseg()           # 以默认配置加载模型
text = seg.cut('并广泛动员社会各方面的力量')  # 进行分词
print(text)

['并', '广泛', '动员', '社会', '各', '方面', '的', '力量']

import pkuseg
seg = pkuseg.pkuseg(postag=True)  # 开启词性标注功能
text = seg.cut('并广泛动员社会各方面的力量')    # 进行分词和词性标注
print(text)

[('并', 'c'), ('广泛', 'ad'), ('动员', 'v'), ('社会', 'n'), ('各', 'r'), ('方面', 'n'), ('的', 'u'), ('力量', 'n')]