基于LDA算法的Twitter文本分析

import os
import pandas as pd
import numpy as np
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
import plotly as py
import plotly.graph_objs as go

import gensim
from gensim import corpora, models, similarities
import logging
import tempfile

import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords

from string import punctuation
from collections import OrderedDict

import seaborn as sns
import pyLDAvis.gensim  # pip install pyLDAvis
import matplotlib.pyplot as plt
%matplotlib inline

init_notebook_mode(connected=True) # do not miss this line

import warnings

df = pd.read_csv("data.csv",encoding="gb18030") 
df.head()

df.shape  # 数据量

(3218, 5)

df.isnull().sum()

row ID             0
Tweet              0
Time               0
Retweet from    2693
User               0
dtype: int64

df.dtypes  # 转换前

row ID          object
Tweet           object
Time            object
Retweet from    object
User            object
dtype: object

df["Time"] = pd.to_datetime(df["Time"])  # 转换成时间格式  

df.dtypes  # 转换后

row ID                  object
Tweet                   object
Time            datetime64[ns]
Retweet from            object
User                    object
dtype: object

df["Time"] = pd.to_datetime(df['Time'], format='%y-%m-%d %H:%M:%S')
df.head()

df.drop("row ID", axis=1, inplace=True)   

tweetsdata = df["Time"]
tweetsdata

0      2017-09-29 17:39:19
1      2017-09-29 10:44:54
2      2017-09-29 10:39:57
3      2017-09-29 09:56:12
4      2017-09-29 09:19:21
               ...        
3213   2012-11-20 08:52:03
3214   2012-11-20 08:38:31
3215   2012-11-20 08:30:44
3216   2012-11-19 08:59:46
3217   2012-11-16 17:59:47
Name: Time, Length: 3218, dtype: datetime64[ns]

trace = go.Histogram(  # 绘制数据
    x = tweetsdata,  # x 轴的数据
    marker = dict(color="blue"),  # 柱子颜色
    opacity = 0.75  # 透明度设置
)

layout = go.Layout(  # 整体布局
    title = "Tweet Activity Over Years",  # 标题-高-宽-xy轴标题-柱子间隔
    height=450,
    width=1200,
    xaxis=dict(title='Month and year'),
    yaxis=dict(title='Tweet Quantity'),
    bargap=0.2)

data = [trace]

fig = go.Figure(data=data, layout=layout)

fig.show()

corpus = df["Tweet"].tolist()
corpus[:5]

["@MeltingIce Assuming max acceleration of 2 to 3 g's, but in a comfortable direction. Will feel like a mild to moder? https://t.co/fpjmEgrHfC",
 'RT @SpaceX: BFR is capable of transporting satellites to orbit, crew and cargo to the @Space_Station and completing missions to the Moon an?',
 '@bigajm Yup :)',
 'Part 2 https://t.co/8Fvu57muhM',
 'Fly to most places on Earth in under 30 mins and anywhere in under 60. Cost per seat should be? https://t.co/dGYDdGttYd']

import os  
  
TEMP_FOLDER = os.getcwd()  # 当前目录

list1 = ['RT','rt']

stoplist = stopwords.words('english') + list(punctuation) + list1

stoplist[:10]

['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're"]

texts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]
print(texts[0])
['@meltingice', 'assuming', 'max', 'acceleration', '2', '3', "g's,", 'comfortable', 'direction.', 'feel', 'like', 'mild', 'moder?', 'https://t.co/fpjmegrhfc']

dictionary = corpora.Dictionary(texts)
dictionary.save(os.path.join(TEMP_FOLDER, 'elon.dict')) 

dictionary.token2id  # 获取每个单词对应的id序号

{'2': 0,
 '3': 1,
 '@meltingice': 2,
 'acceleration': 3,
 'assuming': 4,
 'comfortable': 5,
 'direction.': 6,
 'feel': 7,
 "g's,": 8,
 'https://t.co/fpjmegrhfc': 9,
 'like': 10,
 'max': 11,
 'mild': 12,
 'moder?': 13,
 '@space_station': 14,
 ......
 }

corpus = [dictionary.doc2bow(text) for text in texts] 
corpus[:2]

[[(0, 1),
  (1, 1),
  (2, 1),
  (3, 1),
  (4, 1),
  (5, 1),
  (6, 1),
  (7, 1),
  (8, 1),
  (9, 1),
  (10, 1),
  (11, 1),
  (12, 1),
  (13, 1)],
 [(14, 1),
  (15, 1),
  (16, 1),
  (17, 1),
  (18, 1),
  (19, 1),
  (20, 1),
  (21, 1),
  (22, 1),
  (23, 1),
  (24, 1),
  (25, 1),
  (26, 1)]]

# 语料库的保存
corpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'corpus.mm'), corpus) 

tfidf = models.TfidfModel(corpus)  # 1-模型初始化

corpus_tfidf = tfidf[corpus]  # 2-基于tfidf模型将语料转成向量

total_topics = 5  # 设置5个主题

lda = models.LdaModel(corpus,  # 语料
                      id2word=dictionary,  # 单词与序号的对应字典
                      num_topics=total_topics  # 设置主题数
                     )
corpus_lda = lda[corpus_tfidf] 

lda.show_topics(total_topics, 3)

[(0, '0.006*"..." + 0.006*"tesla" + 0.005*"model"'),
 (1, '0.012*"launch" + 0.011*"falcon" + 0.009*"@spacex:"'),
 (2, '0.014*"tesla" + 0.006*"model" + 0.005*"new"'),
 (3, '0.011*"model" + 0.006*"good" + 0.006*"tesla"'),
 (4, '0.008*"tesla" + 0.006*"model" + 0.005*"w"')]

data_lda = {i: OrderedDict(lda.show_topic(i,25)) for i in range(total_topics)}      
data_lda

{0: OrderedDict([('...', 0.006462383),
              ('tesla', 0.005584647),
              ('model', 0.0048239143),
              ('new', 0.004051302),
              ('next', 0.003930719),
              ('great', 0.0030215571),
              ('good', 0.002984404),
              ('miles', 0.0029328458),
              ('like', 0.002857408),
              ("i'm", 0.0027793457),
              ('rocket', 0.0025001287),
              ('back', 0.0024146684),
              ('@elonmusk', 0.0023003744),
              ('long', 0.0022880563),
              ('super', 0.0022213901),
              ('@spacex', 0.0022024196),
              ('flight', 0.0021213787),
                 ......
                ])}

df_lda = pd.DataFrame(data_lda)
df_lda.head()

df_lda = df_lda.fillna(0).T
df_lda

g = sns.clustermap(df_lda.corr(), # 相关系数
                center = 0,
                standard_scale = 1,
                cmap = "RdBu",
                metric = "cosine",
                linewidths = 0.75,
                figsize = (10,10)
               )

plt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)
plt.show()

pyLDAvis.enable_notebook()

panel = pyLDAvis.gensim.prepare(lda, corpus_lda, dictionary, mds='tsne')
panel