致CSDN读者的一些话：感恩这十年的陪伴，不负遇见，短暂消失

# encoding:utf-8
"""
By：Easmount CSDN 2021-04-19
"""
import re
import time
import csv
import pandas as pd
import numpy as np

#------------------------------------------------------------------------------
#第一步 读取数据
dd = []   #日期
tt = []   #时间
with open("data.csv", "r", encoding="utf8") as csvfile:
    csv_reader = csv.reader(csvfile)
    k = 0
    for row in csv_reader:
        if k==0:      #跳过标题
            k = k + 1
            continue
        #获取数据 2021-04-08 21:52:21
        value_date = row[4]
        value_time = row[5]
        hour = value_time.split(":")[0]
        hour = int(hour)
        dd.append(row[4])
        tt.append(hour)
        #print(row[4],row[5])
        #print(hour)
        k = k + 1
print(len(tt),len(dd))
print(dd)
print(tt)

#------------------------------------------------------------------------------
#第二步 统计不同小时的个数
from collections import Counter
cnt = Counter(tt)
print(cnt.items())  #dict_items
#字典按键排序
list_time = []
list_tnum = []
for i in sorted(cnt):
    print(i,cnt[i])
    list_time.append(i)
    list_tnum.append(cnt[i])

#------------------------------------------------------------------------------
#第三步 绘制柱状图
import matplotlib.pyplot as plt
N = 24
ind = np.arange(N)
width=0.35
plt.bar(ind, list_tnum, width, color='r', label='hour') 
plt.xticks(ind+width/2, list_time, rotation=40) 
plt.title("The Eastmount's blog is distributed by the hour")  
plt.xlabel('hour')  
plt.ylabel('numbers')  
plt.savefig('Eastmount-01.png',dpi=400)  
plt.show()

#------------------------------------------------------------------------------
#第四步 PyEcharts绘制柱状图

from pyecharts import options as opts
from pyecharts.charts import Bar
bar=(
    Bar()
        .add_xaxis(list_time)
        .add_yaxis("数量", list_tnum, color="blue")
        .set_global_opts(title_opts=opts.TitleOpts(
            title="Eastmount博客按小时分布", subtitle="hour"))
    )
bar.render('01-Eastmount博客按小时分布.html')

# encoding:utf-8
"""
By：Easmount CSDN 2021-04-19
"""
import re
import time
import csv
import pandas as pd
import numpy as np

#------------------------------------------------------------------------------
#第一步 读取数据
dd = []   #日期
tt = []   #时间
with open("data.csv", "r", encoding="utf8") as csvfile:
    csv_reader = csv.reader(csvfile)
    k = 0
    for row in csv_reader:
        if k==0:      #跳过标题
            k = k + 1
            continue
        #获取数据 2021-04-08 21:52:21
        value_date = row[4]
        value_time = row[5]
        hour = value_time.split(":")[0]   #获取小时
        hour = int(hour)
        month = value_date[:7]            #获取月份
        dd.append(month)
        tt.append(hour)
        #print(row[4],row[5])
        #print(hour,month)
        print(month)
        k = k + 1
        #break
print(len(tt),len(dd))
print(dd)
print(tt)

#------------------------------------------------------------------------------
#第二步 统计不同日期的个数
from collections import Counter
cnt = Counter(dd)
print(cnt.items())  #dict_items
#字典按键排序
list_date = []
list_dnum = []
for i in sorted(cnt):
    print(i,cnt[i])
    list_date.append(i)
    list_dnum.append(cnt[i])

#------------------------------------------------------------------------------
#第三步 PyEcharts绘制柱状图
from pyecharts import options as opts
from pyecharts.charts import Bar
from pyecharts.charts import Line
from pyecharts.commons.utils import JsCode

line = (
    Line()
    .add_xaxis(list_date)
    .add_yaxis('数量', list_dnum, is_smooth=True,
               markline_opts=opts.MarkLineOpts(data=[opts.MarkLineItem(type_="average")]),
               markpoint_opts=opts.MarkPointOpts(data=[opts.MarkPointItem(type_="max"),
                                                       opts.MarkPointItem(type_="min")]))
    # 隐藏数字 设置面积
    .set_series_opts(
        areastyle_opts=opts.AreaStyleOpts(opacity=0.5),
        label_opts=opts.LabelOpts(is_show=False))
    # 设置x轴标签旋转角度
    .set_global_opts(xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-30)), 
                     yaxis_opts=opts.AxisOpts(name='数量', min_=3), 
                     title_opts=opts.TitleOpts(title='Eastmount博客按日期分布'))        
    )
line.render('02-Eastmount博客按日期分布.html')

# encoding:utf-8
"""
By：Easmount CSDN 2021-04-19
"""
import re
import time
import csv
import pandas as pd
import numpy as np
import datetime

#定义星期函数
def get_week_day(date):
    week_day_dict = {
        0 : '星期一',
        1 : '星期二',
        2 : '星期三',
        3 : '星期四',
        4 : '星期五',
        5 : '星期六',
        6 : '星期天'
    }
    day = date.weekday()
    return week_day_dict[day]

#------------------------------------------------------------------------------
#第一步 读取数据

dd = []   #日期
tt = []   #时间
ww = []   #星期
with open("data.csv", "r", encoding="utf8") as csvfile:
    csv_reader = csv.reader(csvfile)
    k = 0
    for row in csv_reader:
        if k==0:      #跳过标题
            k = k + 1
            continue
        #获取数据 2021-04-08 21:52:21
        value_date = row[4]
        value_time = row[5]
        hour = value_time.split(":")[0]   #获取小时
        hour = int(hour)
        month = value_date[:7]            #获取月份
        dd.append(month)
        tt.append(hour)

        #获取星期
        date = datetime.datetime.strptime(value_date, '%Y-%m-%d').date()
        week = get_week_day(date)
        ww.append(week)
        #print(date,week)
        k = k + 1
print(len(tt),len(dd),len(ww))
print(dd)
print(tt)
print(ww)

#------------------------------------------------------------------------------
#第二步 统计不同日期的个数
from collections import Counter
cnt = Counter(ww)
print(cnt.items())  #dict_items
#字典按键排序
list_date = ['星期一','星期二','星期三','星期四','星期五','星期六','星期天']
list_dnum = [0,0,0,0,0,0,0]

for key,value in cnt.items():
    k = 0
    while k<len(list_date):
        if key==list_date[k]:
            list_dnum[k] = value
            break
        k = k + 1
print(list_date,list_dnum)

#------------------------------------------------------------------------------
#第三步 PyEcharts绘制柱状图
from pyecharts import options as opts
from pyecharts.charts import Bar
from pyecharts.charts import Line
from pyecharts.commons.utils import JsCode
bar=(
    Bar()
        .add_xaxis(list_date)
        .add_yaxis("数量", list_dnum, color='pink')
        .set_global_opts(title_opts=opts.TitleOpts(
            title="Eastmount博客按星期分布", subtitle="week"))
    )
bar.render('03-Eastmount博客按星期分布.html')

# coding=utf-8
"""
By：Easmount CSDN 2021-04-19
"""
import jieba
import re
import time
import csv
from collections import Counter

#------------------------------------中文分词----------------------------------
cut_words = ""
all_words = ""
stopwords = ["[", "]", "）", "（", ")", "(", "【", "】",
             ".", "、", "-", "—", ":", "：", "《", "》",
             "的", "和", "之", "及", "“", "”", "？", "?"]

#导入自定义词典
#jieba.load_userdict("dict.txt")

f = open('06-data-fenci.txt', 'w')

with open("data.csv", "r", encoding="utf8") as csvfile:
    csv_reader = csv.reader(csvfile)
    k = 0
    for row in csv_reader:
        if k==0:      #跳过标题
            k = k + 1
            continue
        #获取数据
        title = row[1]
        title = title.strip('\n')
        #print(title)
        #分词
        cut_words = ""
        seg_list = jieba.cut(title,cut_all=False)
        for seg in seg_list:
            if seg not in stopwords:
                cut_words += seg + " "
        #cut_words = (" ".join(seg_list))
        f.write(cut_words+"\n")
        all_words += cut_words
        k = k + 1
f.close()
#输出结果
all_words = all_words.split()
print(all_words)

#------------------------------------词频统计---------------------------------- 
c = Counter()
for x in all_words:
    if len(x)>1 and x != '\r\n':
        c[x] += 1
#输出词频最高的前10个词
print('\n词频统计结果：')
for (k,v) in c.most_common(10):
    print("%s:%d"%(k,v))
#存储数据
name ="06-data-word.csv"
fw = open(name, 'w', encoding='utf-8')
i = 1
for (k,v) in c.most_common(len(c)):
    fw.write(str(i)+','+str(k)+','+str(v)+'\n')
    i = i + 1
else:
    print("Over write file!")
    fw.close()

# coding=utf-8
"""
By：Easmount CSDN 2021-04-19
"""
import jieba
import re
import time
import csv
from collections import Counter

#------------------------------------中文分词----------------------------------
cut_words = ""
all_words = ""
stopwords = ["[", "]", "）", "（", ")", "(", "【", "】",
             "01", "02", "03", "04", "05", "06", "07",
             "08", "09", "什么"]
f = open('06-data-fenci.txt', 'w')

with open("data.csv", "r", encoding="utf8") as csvfile:
    csv_reader = csv.reader(csvfile)
    k = 0
    for row in csv_reader:
        if k==0:      #跳过标题
            k = k + 1
            continue
        #获取数据
        title = row[1]
        title = title.strip('\n')
        #print(title)
        #分词
        cut_words = ""
        seg_list = jieba.cut(title,cut_all=False)
        for seg in seg_list:
            if seg not in stopwords:
                cut_words += seg + " "
        #cut_words = (" ".join(seg_list))
        f.write(cut_words+"\n")
        all_words += cut_words
        k = k + 1
f.close()
#输出结果
all_words = all_words.split()
print(all_words)

#------------------------------------词频统计---------------------------------- 
c = Counter()
for x in all_words:
    if len(x)>1 and x != '\r\n':
        c[x] += 1
#输出词频最高的前10个词
print('\n词频统计结果：')
for (k,v) in c.most_common(10):
    print("%s:%d"%(k,v))
#存储数据
name ="06-data-word.csv"
fw = open(name, 'w', encoding='utf-8')
i = 1
for (k,v) in c.most_common(len(c)):
    fw.write(str(i)+','+str(k)+','+str(v)+'\n')
    i = i + 1
else:
    print("Over write file!")
    fw.close()

#------------------------------------词云分析----------------------------------
from pyecharts import options as opts
from pyecharts.charts import WordCloud
from pyecharts.globals import SymbolType

# 生成数据 word = [('A',10), ('B',9), ('C',8)] 列表+Tuple
words = []
for (k,v) in c.most_common(200):
    # print(k, v)
    words.append((k,v))
# 渲染图
def wordcloud_base() -> WordCloud:
    c = (
        WordCloud()
        .add("", words, word_size_range=[20, 40], shape='diamond') #shape=SymbolType.ROUND_RECT
        .set_global_opts(title_opts=opts.TitleOpts(title='Eastmount十年博客词云图'))
    )
    return c

# 生成图
wordcloud_base().render('05-Eastmount十年博客词云图.html')

#coding: utf-8
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

#---------------------  第一步 读取数据(已分词)  ----------------------
corpus = []

# 读取预料 一行预料为一个文档
for line in open('06-data-fenci.txt', 'r').readlines():
    corpus.append(line.strip())
        
#-----------------------  第二步 计算TF-IDF值  ----------------------- 
# 设置特征数
n_features = 2000
tf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',
                                max_features=n_features,
                                stop_words=['的','或','等','是','有','之','与','可以','还是','这里',
                                            '一个','和','也','被','吗','于','中','最','但是','大家',
                                            '一下','几天','200','还有','一看','300','50','哈哈哈哈',
                                             '“','”','。','，','？','、','；','怎么','本来','发现',
                                             'and','in','of','the','我们','一直','真的','18','一次',
                                           '了','有些','已经','不是','这么','一一','一天','这个','这种',
                                           '一种','位于','之一','天空','没有','很多','有点','什么','五个',
                                           '特别'],
                                max_df = 0.99,
                                min_df = 0.002) #去除文档内出现几率过大或过小的词汇
tf = tf_vectorizer.fit_transform(corpus)
print(tf.shape)
print(tf)

#-------------------------  第三步 LDA分析  ------------------------ 
from sklearn.decomposition import LatentDirichletAllocation
# 设置主题数
n_topics = 4
lda = LatentDirichletAllocation(n_components=n_topics,
                                max_iter=100,
                                learning_method='online',
                                learning_offset=50,
                                random_state=0)
lda.fit(tf)
# 显示主题数 model.topic_word_
print(lda.components_)
# 几个主题就是几行 多少个关键词就是几列 
print(lda.components_.shape)                         
# 计算困惑度
print(u'困惑度：')
print(lda.perplexity(tf,sub_sampling = False))

# 主题-关键词分布
def print_top_words(model, tf_feature_names, n_top_words):
    for topic_idx,topic in enumerate(model.components_):  # lda.component相当于model.topic_word_
        print('Topic #%d:' % topic_idx)
        print(' '.join([tf_feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]))
        print("")
# 定义好函数之后 暂定每个主题输出前20个关键词
n_top_words = 20                                       
tf_feature_names = tf_vectorizer.get_feature_names()
# 调用函数
print_top_words(lda, tf_feature_names, n_top_words)

#------------------------  第四步 可视化分析  ------------------------- 
import pyLDAvis
import pyLDAvis.sklearn

#pyLDAvis.enable_notebook()
data = pyLDAvis.sklearn.prepare(lda,tf,tf_vectorizer)
print(data)
#显示图形
pyLDAvis.show(data)
pyLDAvis.save_json(data,' 06-fileobj.html')

# -*- coding: utf-8 -*-
import os
import codecs
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import jieba
from sklearn import metrics
from sklearn.metrics import silhouette_score
from array import array
from numpy import *
from pylab import mpl
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
import matplotlib as mpl
from scipy.cluster.hierarchy import ward, dendrogram

#---------------------------------------加载语料-------------------------------------
text = open('06-data-fenci.txt').read()
print(text)
list1=text.split("\n")
print(list1)
print(list1[0])
print(list1[1])
mytext_list=list1

#控制显示数量
count_vec = CountVectorizer(min_df=20, max_df=1000)  #最大值忽略
xx1 = count_vec.fit_transform(list1).toarray()
word=count_vec.get_feature_names() 
print("word feature length: {}".format(len(word)))
print(word)
print(xx1)
print(type(xx1))
print(xx1.shape)
print(xx1[0])

#---------------------------------------层次聚类-------------------------------------
titles = word
#dist = cosine_similarity(xx1)

mpl.rcParams['font.sans-serif'] = ['SimHei']

df = pd.DataFrame(xx1)
print(df.corr())
print(df.corr('spearman'))
print(df.corr('kendall'))
dist = df.corr()
print (dist)
print(type(dist))
print(dist.shape)

#define the linkage_matrix using ward clustering pre-computed distances
linkage_matrix = ward(dist) 
fig, ax = plt.subplots(figsize=(8, 12)) # set size
ax = dendrogram(linkage_matrix, orientation="right",
                p=20, labels=titles, leaf_font_size=12
                ) #leaf_rotation=90., leaf_font_size=12.
#show plot with tight layout
plt.tight_layout() 
#save figure as ward_clusters
plt.savefig('07-KH.png', dpi=200)
plt.show()

# -*- coding: utf-8 -*-
"""
@author: eastmount CSDN 2020-04-20
"""
import pandas as pd
import numpy as np
import codecs
import networkx as nx
import matplotlib.pyplot as plt
import csv
from scipy.sparse import coo_matrix
 
#---------------------------第一步:读取数据-------------------------------
word = [] #记录关键词
f = open("06-data-fenci.txt", encoding='gbk')            
line = f.readline()           
while line:
    #print line
    line = line.replace("\n", "") #过滤换行
    line = line.strip('\n') 
    for n in line.split(' '):
        #print n
        if n not in word:
            word.append(n)
    line = f.readline()
f.close()
print(len(word)) #关键词总数 2913

#--------------------------第二步 计算共现矩阵----------------------------
a = np.zeros([2,3])
print(a)

#共现矩阵
#word_vector = np.zeros([len(word),len(word)], dtype='float16')

#MemoryError：矩阵过大汇报内存错误 采用coo_matrix函数解决该问题
print(len(word))
#类型<type 'numpy.ndarray'>
word_vector = coo_matrix((len(word),len(word)), dtype=np.int8).toarray() 
print(word_vector.shape)

f = open("06-data-fenci.txt", encoding='gbk')  
line = f.readline()           
while line:
    line = line.replace("\n", "") #过滤换行
    line = line.strip('\n') #过滤换行
    nums = line.split(' ')

    #循环遍历关键词所在位置 设置word_vector计数
    i = 0
    j = 0
    while i<len(nums):         #ABCD共现 AB AC AD BC BD CD加1
        j = i + 1
        w1 = nums[i]           #第一个单词
        while j<len(nums):
            w2 = nums[j]       #第二个单词
            #从word数组中找到单词对应的下标
            k = 0
            n1 = 0
            while k<len(word):
                if w1==word[k]:
                    n1 = k
                    break
                k = k +1
            #寻找第二个关键字位置
            k = 0
            n2 = 0
            while k<len(word):
                if w2==word[k]:
                    n2 = k
                    break
                k = k +1
            #重点: 词频矩阵赋值 只计算上三角
            if n1<=n2:
                word_vector[n1][n2] = word_vector[n1][n2] + 1
            else:
                word_vector[n2][n1] = word_vector[n2][n1] + 1
            #print(n1, n2, w1, w2)
            j = j + 1
        i = i + 1
    #读取新内容
    line = f.readline()
    #print("next line")
f.close()
print("over computer")


#--------------------------第三步  CSV文件写入--------------------------
c = open("word-word-weight.csv","w", encoding='utf-8', newline='')    #解决空行
#c.write(codecs.BOM_UTF8)                                 #防止乱码
writer = csv.writer(c)                                    #写入对象
writer.writerow(['Word1', 'Word2', 'Weight'])

i = 0
while i<len(word):
    w1 = word[i]
    j = 0 
    while j<len(word):
        w2 = word[j]
        #判断两个词是否共现 共现词频不为0的写入文件
        if word_vector[i][j]>0:
            #写入文件
            templist = []
            templist.append(w1)
            templist.append(w2)
            templist.append(str(int(word_vector[i][j])))
            #print templist
            writer.writerow(templist)
        j = j + 1
    i = i + 1
c.close()

# -*- coding: utf-8 -*-
from snownlp import SnowNLP
import codecs
import os

#获取情感分数
source = open("06-data-fenci.txt", "r", encoding='gbk')
fw = open("09-result.txt", "w", encoding="gbk")
line = source.readlines()
sentimentslist = []
for i in line:
    s = SnowNLP(i)
    #print(s.sentiments)
    sentimentslist.append(s.sentiments)

#区间转换为[-0.5, 0.5]
result = []
i = 0
while i<len(sentimentslist):
    result.append(sentimentslist[i]-0.5)
    fw.write(str(sentimentslist[i]-0.5)+"\n")
    print(sentimentslist[i]-0.5, line[i].strip("\n"))
    i = i + 1
fw.close()

#可视化画图
import matplotlib.pyplot as plt
import numpy as np
plt.plot(np.arange(0, 598, 1), result, 'k-')
plt.xlabel('Number')
plt.ylabel('Sentiment')
plt.title('Analysis of Sentiments')
plt.show()

option = {
    series: {
        type: 'sankey',
        layout:'none',
        focusNodeAdjacency: 'allEdges',
        data: [
        {
            name: 'T1-竞争情报'
        },{
            name: 'T1-企业'
        },{
            name: 'T1-企业管理'
        }, {
            name: 'T1-情报研究'
        },{
            name: 'T1-竞争对手'
        },{
            name: 'T1-情报工作'
        },{
            name: 'T1-市场经济'
        },{
            name: 'T2-竞争情报'
        },{
            name: 'T2-企业'
        },{
            name: 'T2-企业管理'
        },{
            name: 'T2-竞争情报系统'
        },{
            name: 'T2-竞争对手'
        },{
            name: 'T2-知识管理'
        },{
            name: 'T2-反竞争情报'
        },{
            name: 'T3-竞争情报'
        },{
            name: 'T3-企业'
        },{
            name: 'T3-竞争情报系统'
        },{
            name: 'T3-企业管理'
        },{
            name: 'T3-高校图书馆'
        },{
            name: 'T3-反竞争情报'
        },{
            name: 'T3-知识管理'
        },{
            name: 'T4-竞争情报'
        },{
            name: 'T4-企业'
        },{
            name: 'T4-大数据'
        },{
            name: 'T4-产业竞争情报'
        },{
            name: 'T4-竞争情报系统'
        },{
            name: 'T4-高校图书馆'
        },{
            name: 'T4-企业管理'
        }
        
        ],
        links: [{
            source: 'T1-竞争情报',
            target: 'T2-竞争情报',
            value: 10
        }, {
            source: 'T1-企业',
            target: 'T2-企业',
            value: 7
        }, {
            source: 'T1-企业管理',
            target: 'T2-企业管理',
            value: 6
        },{
            source: 'T1-情报研究',
            target: 'T2-竞争情报',
            value: 5
        },{
            source: 'T1-竞争对手',
            target: 'T2-竞争对手',
            value: 5
        },{
            source: 'T1-情报工作',
            target: 'T2-竞争情报',
            value: 3
        },{
            source: 'T1-市场经济',
            target: 'T2-竞争情报',
            value: 3
        },{
            source: 'T1-竞争情报',
            target: 'T2-竞争情报系统',
            value: 5
        },{
            source: 'T1-竞争情报',
            target: 'T2-竞争情报系统',
            value: 3
        },{
            source: 'T1-竞争情报',
            target: 'T2-知识管理',
            value: 3
        },{
            source: 'T1-竞争情报',
            target: 'T2-反竞争情报',
            value: 3
        },
        
        {
            source: 'T2-竞争情报',
            target: 'T3-竞争情报',
            value: 10
        },{
            source: 'T2-企业',
            target: 'T3-企业',
            value: 7
        },{
            source: 'T3-竞争情报系统',
            target: 'T4-竞争情报',
            value: 3
        },{
            source: 'T2-企业管理',
            target: 'T3-企业管理',
            value: 6
        },{
            source: 'T2-竞争情报系统',
            target: 'T3-竞争情报系统',
            value: 5
        },{
            source: 'T2-竞争对手',
            target: 'T3-竞争情报',
            value: 5
        },{
            source: 'T2-知识管理',
            target: 'T3-知识管理',
            value: 3
        },{
            source: 'T2-反竞争情报',
            target: 'T3-反竞争情报',
            value: 3
        },{
            source: 'T2-竞争情报',
            target: 'T3-高校图书馆',
            value: 4
        },
        
        {
            source: 'T3-竞争情报',
            target: 'T4-竞争情报',
            value: 10
        },{
            source: 'T3-企业',
            target: 'T4-企业',
            value: 7
        },{
            source: 'T3-竞争情报',
            target: 'T4-大数据',
            value: 5
        },{
            source: 'T3-竞争情报',
            target: 'T4-产业竞争情报',
            value: 5
        },{
            source: 'T3-竞争情报系统',
            target: 'T4-竞争情报系统',
            value: 6
        },{
            source: 'T3-企业管理',
            target: 'T4-企业管理',
            value: 4
        },
        {
            source: 'T3-高校图书馆',
            target: 'T4-高校图书馆',
            value: 4
        },{
            source: 'T3-反竞争情报',
            target: 'T4-竞争情报',
            value: 3
        },{
            source: 'T3-知识管理',
            target: 'T4-竞争情报',
            value: 2
        }
        
        ]
    }
};