文本分类实战: 机器学习vs深度学习算法对比（附代码）

2,合晟资产是一家专注于股票、债券等二级市场投资，为合格投资者提供专业资产管理服务的企业。公司业务范围包括资产管理、投资咨询和投资顾问服务。公司管理的私募基金产品主要包括股票型、债券型资产管理计划或证券投资基金，管理总资产规模80亿元左右。根据中国证券投资基金业协会数据，公司管理的私募证券投资基金（顾问管理）类规模较大，公司管理规模处于50亿元以上的第一梯队。
2,公司的主营业务为向中小微企业、个体工商户、农户等客户提供贷款服务，自设立以来主营业务未发生过变化。

import numpy as npwith open('output/random_out.csv', 'w') as f:    for i in range(1, 2382):
        f.write(str(i))
        f.write(',')
        aa = np.random.random()
        b = 0
        if aa <= 0.25:
            b = 3
        elif aa <= 0.5:
            b = 4
        elif aa <= 0.7:
            b =6
        elif aa <= 0.775:
            b=7
        elif aa <= 0.825:
            b = 5
        elif aa <= 0.875:
            b = 8
        elif aa <= 0.925:
            b = 10
        elif aa <= 0.95:
            b = 11
        elif aa <= 0.975:
            b = 2
        elif aa <= 1:
            b = 9
        f.write(str(b))
        f.write('\n')

feature_words = [[u'物业管理', u'物业', u'房地产', u'顾问', u'中介', u'住宅', u'商业', u'开发商', u'招商', u'营销策划'],
                 [u'私募', u'融资', u'金融', u'贷款', u'基金', u'股权', u'资产', u'小额贷款', u'投资', u'担保'],
                 [u'软件', u'互联网', u'平台', u'信息化', u'软件开发', u'数据', u'移动', u'信息', u'系统集成', u'运营'],
                 [u'制造', u'安装', u'设备', u'施工', u'机械', u'工程', u'自动化', u'工业', u'设计', u'装备'],
                 [u'药品', u'医药', u'生物', u'原料药', u'药物', u'试剂', u'GMP', u'片剂', u'制剂', u'诊断'],
                 [u'材料', u'制品', u'塑料', u'环保', u'新型', u'化学品', u'改性', u'助剂', u'涂料', u'原材料'],
                 [u'养殖', u'农业', u'种植', u'食品', u'加工', u'龙头企业', u'产业化', u'饲料', u'基地', u'深加工'],
                 [u'医疗器械', u'医疗', u'医院', u'医用', u'康复', u'治疗', u'医疗机构', u'临床', u'护理'],
                 [u'汽车', u'零部件', u'发动机', u'整车', u'模具', u'C36', u'配件', u'总成', u'车型'],
                 [u'媒体', u'制作', u'策划', u'广告', u'传播', u'创意', u'发行', u'影视', u'电影', u'文化'],
                 [u'运输', u'物流', u'仓储', u'货物运输', u'货运', u'装卸', u'配送', u'第三方', u'应链', u'集装箱']]

documents_feature =json.load(open('tmp/documents_feature.txt', 'r'))
test_documents_feature = json.load(open('tmp/test_documents_feature.txt', 'r'))

print "开始训练分类器"classifier = nltk.NaiveBayesClassifier.train(documents_feature)#classifier = nltk.DecisionTreeClassifier.train(documents_feature)#classifier = SklearnClassifier(SVC(), sparse=False).train(documents_feature[:4000])#test_error = nltk.classify.accuracy(classifier, documents_feature)#print "test_error:", test_error#classifier.show_most_informative_features(20)results = classifier.classify_many([fs for fs in test_documents_feature])with open('output/TFIDF_out.csv', 'w') as f:    for i in range(2381):
        f.write(str(i+1))
        f.write(',')
        f.write(str(results[i]+1))
        f.write('\n')

result = []
qq = 0with open('data/testing.csv', u'r') as f:    for line in f:
        content = ""
        for aa in line.split(',')[1:]:
            content += aa
        word_list, word_set = jieba_fenci(content, stopwords_list)
        label = 2
        count = 0
        for i, cla in enumerate(feature_words):
            tmp = 0
            for word in word_list:                if word in cla:
                    tmp += 1
            if tmp > count:
                count = tmp
                label = i+1
        if count == 0:
            qq += 1
        result.append(label)

# -*- coding: utf-8 -*-import xgboost as xgbimport csvimport jieba#jieba.load_userdict('wordDict.txt')import numpy as npfrom sklearn.feature_extraction.text import CountVectorizerfrom sklearn.feature_extraction.text import TfidfTransformer# 读取训练集def readtrain(path):
    with open(path, 'rb') as csvfile:
        reader = csv.reader(csvfile)
        column1 = [row for row in reader]
    content_train = [i[1] for i in column1] # 第一列为文本内容，并去除列名
    opinion_train = [int(i[0])-1 for i in column1] # 第二列为类别，并去除列名
    print '训练集有 %s 条句子' % len(content_train)
    train = [content_train, opinion_train]    return traindef stop_words():
    stop_words_file = open('stop_words_ch.txt', 'r')
    stopwords_list = []    for line in stop_words_file.readlines():
        stopwords_list.append(line.decode('gbk')[:-1])    return stopwords_list# 对列表进行分词并用空格连接def segmentWord(cont):
    stopwords_list = stop_words()
    c = []    for i in cont:
        text = ""
        word_list = list(jieba.cut(i, cut_all=False))        for word in word_list:            if word not in stopwords_list and word != '\r\n':
                text += word
                text += ' '
        c.append(text)    return cdef segmentWord1(cont):
    c = []    for i in cont:
        a = list(jieba.cut(i))
        b = " ".join(a)
        c.append(b)    return c

train = readtrain('data/training.csv')
train_content = segmentWord1(train[0])
train_opinion = np.array(train[1])     # 需要numpy格式print "train data load finished"test = readtrain('data/testing.csv')
test_content = segmentWord(test[0])print 'test data load finished'vectorizer = CountVectorizer()
tfidftransformer = TfidfTransformer()
tfidf = tfidftransformer.fit_transform(vectorizer.fit_transform(train_content))
weight = tfidf.toarray()print tfidf.shape
test_tfidf = tfidftransformer.transform(vectorizer.transform(test_content))
test_weight = test_tfidf.toarray()print test_weight.shape


dtrain = xgb.DMatrix(weight, label=train_opinion)
dtest = xgb.DMatrix(test_weight)  # label可以不要，此处需要是为了测试效果param = {'max_depth':6, 'eta':0.5, 'eval_metric':'merror', 'silent':1, 'objective':'multi:softmax', 'num_class':11}  # 参数evallist  = [(dtrain,'train')]  # 这步可以不要，用于测试效果num_round = 100  # 循环次数bst = xgb.train(param, dtrain, num_round, evallist)
preds = bst.predict(dtest)with open('XGBOOST_OUTPUT.csv', 'w') as f:    for i, pre in enumerate(preds):
        f.write(str(i + 1))
        f.write(',')
        f.write(str(int(pre) + 1))
        f.write('\n')