关于词云可视化笔记六（射雕英雄传前10章可视化分析）

# coding: UTF-8
import jieba.posseg as pseg
import pkuseg
import os
import sys
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
import pprint
from scipy.interpolate import spline
from pyecharts import ThemeRiver
from collections import Counter
import matplotlib.pyplot as plt
from matplotlib import cm
from numpy import matrix
import pylab
import numpy as np
def draw_river(data,xlabels,ylabels,title='',step=300):
    # X标签 行，即章节
    # Y标签 列，即词汇
    # 数据 即词频，需要转置后才能应用
    #获取y轴数量
    ylen=len(ylabels)
    #初始化一个X轴的序列numpy数组，默认为[0 1 2 len(xlabel)]
    initX = np.array(range(len(xlabels)))
    #linspace用于创建一个是等差数列的一维数组，最小值是0，最大值是X轴长度，
    xnew=np.linspace(initX.min(), initX.max(), step)
    #创建一个numpy空的二维数组newdata，以便存储转换后的data值
    newdata=np.empty(shape=[0,step])
    #spline只能应用于一维数组，所需需要分行读取
    for datarow in range(ylen):
        power_smooth = spline(initX, data[datarow], xnew)
        #将一维numpy数组变为二维数据
        middata = power_smooth[np.newaxis, :]
        #将二维数组添加到最终的数组中
        newdata=np.append(newdata,middata,axis=0)
    pylab.mpl.rcParams['font.sans-serif'] = ['SimHei']  # 防止中文乱码
    pylab.mpl.rcParams['axes.unicode_minus'] = False  # 防止中文乱码
    figure = plt.figure(facecolor='w', figsize=(11, 7))
    ax = figure.add_subplot(1, 1, 1)
    #用stackplot绘制新的图形
    ax.stackplot(xnew, newdata,labels=ylabels, baseline='wiggle')
    #ax.axes.set_yticks(range(len(ylabels)))
    #ax.axes.set_yticklabels(ylabels)
    ax.axes.set_xticks(range(len(xlabels)))
    ax.axes.set_xticklabels(xlabels)
    ax.set_title(title)
    plt.legend(loc='best', numpoints=1, fontsize=8, bbox_to_anchor=(0., 1.02, 1., .102),
               ncol=10, mode="expand", borderaxespad=0.)
    #ax.set_title(title)
    plt.show()

#文本词频可视化图表heatmap风格
def draw_heatmap(data, xlabels, ylabels):
    pylab.mpl.rcParams['font.sans-serif'] = ['SimHei']  # 防止中文乱码
    pylab.mpl.rcParams['axes.unicode_minus'] = False  # 防止中文乱码
    vmin=np.amin(matrix(data))
    vmax = np.amax(matrix(data))
    cmap = cm.Blues
    figure = plt.figure(facecolor='w', figsize=(11, 7))
    ax = figure.add_subplot(1, 1, 1)
    ax.set_yticks(range(len(ylabels)))
    ax.set_yticklabels(ylabels)
    ax.set_xticks(range(len(xlabels)))
    ax.set_xticklabels(xlabels)
    map = ax.imshow(data, interpolation='nearest', cmap=cmap, aspect='auto', vmin=vmin, vmax=vmax)
    cb = plt.colorbar(mappable=map, cax=None, ax=None, shrink=0.5)
    plt.xticks(rotation=90)  # 将字体进行旋转
    plt.yticks(rotation=360)
    plt.yticks(fontsize=8)
    plt.show()

#获取指定目录文件列表
def getfilelist(path):
    #访问目标目录，获得文件列表
    filelist = []
    files = os.listdir(path)
    #遍历文件夹，判断是否目录还是文件，并加入filelist列表
    for file in files:
        pathfile = os.path.join(path, file)
        if os.path.isdir(pathfile):
            pass
        else:
            filelist.append(pathfile)
    return filelist

#获取停用词库字典的停用词列表
def getstopwords(stopfilename): # 获取停用词表
    stopwords=open(stopfilename,encoding='utf-8',errors='ignore').read()
    stopwordlist=stopwords.split()
    return stopwordlist

#获取人员列表
def getsonlyuserdict(text,userdicts): # 获取人员列表
    new_text = []
    for w in text:
        if w in userdicts:
            new_text.append(w)
    return new_text

#获取停用词库字典的停用词列表
def getsuserdict(dictfilename): # 获取停用词表
    userdicts=open(dictfilename,encoding='utf-8',errors='ignore').read()
    userdictlist=userdicts.split()
    return userdictlist

#获取专有词库
def getprivatedict(dictfilename): # 获取停用词表
    userdicts=open(dictfilename,encoding='utf-8',errors='ignore').read()
    userdictlist=userdicts.split()
    return userdictlist

def cutfilewithdict(pathfile,lexicon):
    f=open(pathfile,encoding='utf-8',errors='ignore').read()
    seg = pkuseg.PKUSeg(user_dict=lexicon)
    # 进行分词
    text = seg.cut(f)
    return text

def cutfilefilterstopword(text,stopwordlist):
    new_text = []
    for w in text:
        if w not in stopwords:
            new_text.append(w)
    return new_text

if __name__ == "__main__":
    path1='D:\Python36\Coding\PycharmProjects\\ttt\射雕英雄传'
    path2 = 'D:\Python36\Coding\PycharmProjects\\ttt\神雕侠侣'
    path3 = 'D:\Python36\Coding\PycharmProjects\\ttt\倚天屠龙记'
    pathfile='D:\Python36\Coding\PycharmProjects\\ttt\射雕英雄传\\1.txt'
    stopfilename='D:\Python36\Coding\PycharmProjects\\ttt\stopword.txt'
    dictfilename='D:\Python36\Coding\PycharmProjects\\ttt\射雕三部曲.txt'
    userfilename = 'D:\Python36\Coding\PycharmProjects\\ttt\射雕三部曲.txt'

    #获取自定义专有词库
    #获取停用词词典，通过网上收集
    #获取小说重点待解析内容
    userdicts=getsuserdict(dictfilename)
    stopwords = getstopwords(stopfilename)
    privateuser=getprivatedict(userfilename)
    #-------------------三步骤---------------------
    #通过自定义词库进行分词
    #过滤停用词词典的词汇
    #获取目标想要的词汇
    #originaltext=cutfilewithdict(pathfile,userdicts)
    #removestopword=cutfilefilterstopword(originaltext,stopwords)
    #onlyuser= getsonlyuserdict(removestopword, privateuser)  # 获取人员列表
    # -------------------三步骤---------------------
    #定义该章节内的top关键字的分词结果
    # 每个章节一个一维数组
    # 格式如[['小龙女', '丘处机', '丘处机', '小龙女', '杨过'],['丘处机', '丘处机', '小龙女']]
    top50userlist=[]
    # 定义该章节内的前topN词频统计结果
    # 每个章节一个一维数组
    #格式如[[('郭襄', 225), ('无色', 76), ('何足道', 53), ('矮老者', 1)], [('张三丰', 113), ('张翠山', 101)]]
    chaptertop50userlist=[]
    # 按词频分析结构定义一维数组
    #每个章节一条记录
    #格式如['小龙女 丘处机 何足道', '何足道 张君宝 张三丰']
    top50list=[]
    i=0
    for pathfile in getfilelist(path1):
        originaltext = cutfilewithdict(pathfile, userdicts)
        removestopword = cutfilefilterstopword(originaltext, stopwords)
        onlyuser = getsonlyuserdict(removestopword, privateuser)  # 获取人员列表
        top50list.append( ' '.join(onlyuser))
        top50userlist.append(onlyuser)
        counter=Counter(onlyuser)
        chaptertop50userlist.append(counter.most_common(100))
    # 将两维数组转一维数组
    # 格式如 ['小龙女', '丘处机', '丘处机',.....]
    alltop50users = [y for x in top50userlist for y in x]
    #重新统计词频，并获取总的前50名用户和及次数
    #格式如[('张翠山', 1105), ('殷素素', 539), ('谢逊', 501)]
    counter = Counter(alltop50users)
    top50user=counter.most_common(50)
    #获取全部章节的top50名的用户
    #格式如 ['张翠山', '殷素素', '谢逊', '郭襄']
    top50 = [x[0] for x in top50user]   #统计前50名的用户
    #重新定义新数组，只保留前50用户的分词情况
    newtop50list=[]
    for i in top50list: #循环初始列表，原来有重复
        tempi=i.split(' ')  #将字符串转为数组，进行迭代
        new_text = []
        for w in tempi:     #遍历当前行的字符串数组
            if w in top50:  #判断是否在前50行列，如果是追加到新数组中
                new_text.append(w)
        newtop50list.append(' '.join(new_text)) #再转为数组结构
    # --------------------------词频分析---------------------------
    vectorizer = CountVectorizer()
    # 该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频
    X = vectorizer.fit_transform(newtop50list)
    # 该类会统计每个词语的tf-idf权值
    transformer = TfidfTransformer()
    # 第一个fit_transform是计算tf-idf矩阵
    tfidf = transformer.fit_transform(X)
    # 获取词袋模型中的所有词语
    word = vectorizer.get_feature_names()
    #将tfidf转换为二维的数组
    #tfidfarr = tfidf.toarray()
    # 热力图方式
    xlabels = word
    ylabels = list(range(len(chaptertop50userlist)))
    data = X.T.toarray().tolist()
    draw_heatmap(data, ylabels, xlabels)
    # 转置维stackflow的格式要求，y轴为字符，x轴为章节
    # stackplt方式
    #data = X.T.toarray().tolist()
    draw_river(data, ylabels, xlabels, title='词云河流图', step=300)
    # --------------------------tfidf分析---------------------------
    data = tfidf.T.toarray().tolist()
    draw_heatmap(data, ylabels, xlabels)
    draw_river(data, ylabels, xlabels, title='词云河流图', step=300)