20 newsgroups数据介绍以及文本分类实例简介基本使用将文本转为TF-IDF向量使用贝叶斯进行分类参考

from sklearn.datasets import fetch_20newsgroups
from pprint import pprint
newsgroups_train = fetch_20newsgroups(subset='train')
pprint(list(newsgroups_train.targernames))

['alt.atheism',
 'comp.graphics',
 'comp.os.ms-windows.misc',
 'comp.sys.ibm.pc.hardware',
 'comp.sys.mac.hardware',
 'comp.windows.x',
 'misc.forsale',
 'rec.autos',
 'rec.motorcycles',
 'rec.sport.baseball',
 'rec.sport.hockey',
 'sci.crypt',
 'sci.electronics',
 'sci.med',
 'sci.space',
 'soc.religion.christian',
 'talk.politics.guns',
 'talk.politics.mideast',
 'talk.politics.misc',
 'talk.religion.misc']

print(newsgroups_train.filenames.shape) # (11314,)
print(newsgroups_train.target.shape) # (11314,)

print(newsgroups_train.target[:10]) # [ 7  4  4  1 14 16 13  3  2  4]
print(newsgroups_train['data'][:2]) # 前三篇文章["From: lerxst@wam.umd.edu (where's my thin...

fetch_20newsgroups(data_home=None, # 文件下载的路径
                   subset='train', # 加载那一部分数据集 train/test
                   categories=None, # 选取哪一类数据集[类别列表]，默认20类
                   shuffle=True,  # 将数据集随机排序
                   random_state=42, # 随机数生成器
                   remove=(), # ('headers','footers','quotes') 去除部分文本
                   download_if_missing=True # 如果没有下载过，重新下载
                   )

from sklearn.feature_extraction.text import TfidfVectorizer
# 我们选取三类作为实验
categories = ['alt.atheism', 'talk.religion.misc','comp.graphics', 'sci.space']
# 加载数据集
newsgroups_train = fetch_20newsgroups(subset='train',categories=categories)
# 提取tfidf特征
vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform(newsgroups_train.data)
print(vectors.shape)
print(vectors.nnz / float(vectors.shape[0]))

# 输出
(2034, 34118)
159.0132743362832

from sklearn.feature_extraction.text import TfidfVectorizer
# 我们选取三类作为实验
categories = ['alt.atheism', 'talk.religion.misc','comp.graphics', 'sci.space']
# 加载数据集
newsgroups_train = fetch_20newsgroups(subset='train',categories=categories)
# 提取tfidf特征
vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform(newsgroups_train.data)
print(vectors.shape)
print(vectors.nnz / float(vectors.shape[0]))

# MultinomialNB实现文本分类
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score,f1_score
# 加载测试集
newsgroups_test=fetch_20newsgroups(subset='test',categories=categories)
# 提取测试集tfidf特征
vectors_test=vectorizer.transform(newsgroups_test.data)
# 训练
clf=MultinomialNB(alpha=0.1)
clf.fit(vectors,newsgroups_train.target)
# 预测
pred=clf.predict(vectors_test)
print(f1_score(newsgroups_test.target,pred,average='macro'))
print(accuracy_score(newsgroups_test.target,pred))

# 输出
f1_score: 0.8823530044163621
accuracy: 0.8965262379896526