文本分类（六）：使用fastText对文本进行分类--小插曲

pip install fasttext

import jieba

basedir = "/home/li/corpus/news/" #这是我的文件地址，需跟据文件夹位置进行更改
dir_list = ['affairs','constellation','economic','edu','ent','fashion','game','home','house','lottery','science','sports','stock']
##生成fastext的训练和测试数据集

ftrain = open("news_fasttext_train.txt","w")
ftest = open("news_fasttext_test.txt","w")

num = -1
for e in dir_list:
    num += 1
    indir = basedir + e + '/'
    files = os.listdir(indir)
    count = 0
    for file in files:
        count += 1            
        filepath = indir + file
        with open(filepath,'r') as fr:
            text = fr.read()
        text = text.decode("utf-8").encode("utf-8")
        seg_text = jieba.cut(text.replace("\t"," ").replace("\n"," "))
        outline = " ".join(seg_text)
        outline = outline.encode("utf-8") + "\t__label__" + e + "\n"
#         print outline
#         break

        if count < 10000:
            ftrain.write(outline)
            ftrain.flush()
            continue
        elif count  < 20000:
            ftest.write(outline)
            ftest.flush()
            continue
        else:
            break

ftrain.close()
ftest.close()

# _*_coding:utf-8 _*_
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
import fasttext
#训练模型
classifier = fasttext.supervised("news_fasttext_train.txt","news_fasttext.model",label_prefix="__label__")

#load训练好的模型
#classifier = fasttext.load_model('news_fasttext.model.bin', label_prefix='__label__')

#测试模型
result = classifier.test("news_fasttext_test.txt")
print result.precision
print result.recall

0.92240420242
0.92240420242

labels_right = []
texts = []
with open("news_fasttext_test.txt") as fr:
    lines = fr.readlines()
for line in lines:
    labels_right.append(line.split("\t")[1].rstrip().replace("__label__",""))
    texts.append(line.split("\t")[0].decode("utf-8"))
#     print labels
#     print texts
#     break
labels_predict = [e[0] for e in classifier.predict(texts)] #预测输出结果为二维形式
# print labels_predict

text_labels = list(set(labels_right))
text_predict_labels = list(set(labels_predict))
print text_predict_labels
print text_labels

A = dict.fromkeys(text_labels,0)  #预测正确的各个类的数目
B = dict.fromkeys(text_labels,0)   #测试数据集中各个类的数目
C = dict.fromkeys(text_predict_labels,0) #预测结果中各个类的数目
for i in range(0,len(labels_right)):
    B[labels_right[i]] += 1
    C[labels_predict[i]] += 1
    if labels_right[i] == labels_predict[i]:
        A[labels_right[i]] += 1

print A 
print B
print C
#计算准确率，召回率，F值
for key in B:
    p = float(A[key]) / float(B[key])
    r = float(A[key]) / float(C[key])
    f = p * r * 2 / (p + r)
    print "%s:\tp:%f\t%fr:\t%f" % (key,p,r,f)

[u'affairs', u'fashion', u'lottery', u'house', u'science', u'sports', u'game', u'economic', u'ent', u'edu', u'home', u'constellation', u'stock']
['affairs', 'fashion', 'house', 'sports', 'game', 'economic', 'ent', 'edu', 'home', 'stock', 'science']
{'science': 8415, 'affairs': 8257, 'fashion': 3173, 'house': 9491, 'sports': 9739, 'game': 9506, 'economic': 9235, 'ent': 9665, 'edu': 9491, 'home': 9315, 'stock': 9015}
{'science': 10000, 'affairs': 10000, 'fashion': 3369, 'house': 10000, 'sports': 10000, 'game': 10000, 'economic': 10000, 'ent': 10000, 'edu': 10000, 'home': 10000, 'stock': 10000}
{u'affairs': 8562, u'fashion': 3585, u'lottery': 96, u'science': 9088, u'edu': 10068, u'sports': 10099, u'game': 10151, u'economic': 10131, u'ent': 10798, u'house': 10000, u'home': 10103, u'constellation': 432, u'stock': 10256}

science:    p:0.841500  r:0.925946r:    f:0.881706
affairs:    p:0.825700  r:0.964377r:    f:0.889667
fashion:    p:0.941822  r:0.885077r:    f:0.912568
house:  p:0.949100  r:0.949100r:    f:0.949100
sports: p:0.973900  r:0.964353r:    f:0.969103
game:   p:0.950600  r:0.936459r:    f:0.943477
economic:   p:0.923500  r:0.911559r:    f:0.917490
ent:    p:0.966500  r:0.895073r:    f:0.929416
edu:    p:0.949100  r:0.942690r:    f:0.945884
home:   p:0.931500  r:0.922003r:    f:0.926727
stock:  p:0.901500  r:0.878998r:    f:0.890107