python主题建模可视化LDA和T-SNE交互式可视化

In [1]:
from scipy import sparse as sp

Populating the interactive namespace from numpy and matplotlib

In [2]:
docs = array(p_df['PaperText'])

from nltk.stem.wordnet import WordNetLemmatizerfrom nltk.tokenize import RegexpTokenizer
def docs_preprocessor(docs):tokenizer = RegexpTokenizer(r'\w+')for idx in range(len(docs)):docs[idx] = docs[idx].lower()  # Convert to lowercase.docs[idx] = tokenizer.tokenize(docs[idx])  # Split into words.
# 删除数字，但不要删除包含数字的单词。
docs = [[token for token in doc if not token.isdigit()] for doc in docs]
# 删除仅一个字符的单词。
docs = [[token for token in doc if len(token) > 3] for doc in docs]
# 使文档中的所有单词规则化
lemmatizer = WordNetLemmatizer()docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]
return docs

docs = docs_preprocessor(docs)

from gensim.models import Phrases# 向文档中添加双字母组和三字母组（仅出现10次或以上的文档）。bigram = Phrases(docs, min_count=10)trigram = Phrases(bigram[docs])
for idx in range(len(docs)):for token in bigram[docs[idx]]:if '_' in token:# Token is a bigram, add to document.docs[idx].append(token)for token in trigram[docs[idx]]:if '_' in token:# token是一个二元组，添加到文档中。docs[idx].append(token)
Using TensorFlow backend./opt/conda/lib/python3.6/site-packages/gensim/models/phrases.py:316: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser classwarnings.warn("For a faster implementation, use the gensim.models.phrases.Phraser class")

from gensim.corpora import Dictionary
# 创建文档的字典表示
dictionary = Dictionary(docs)print('Number of unique words in initital documents:', len(dictionary))
# 过滤掉少于10个文档或占文档20％以上的单词。
dictionary.filter_extremes(no_below=10, no_above=0.2)print('Number of unique words after removing rare and common words:', len(dictionary))
Number of unique words in initital documents: 39534Number of unique words after removing rare and common words: 6001

corpus = [dictionary.doc2bow(doc) for doc in docs]

In [8]:



print('Number of unique tokens: %d' % len(dictionary))print('Number of documents: %d' % len(corpus))
Number of unique tokens: 6001Number of documents: 403

from gensim.models import LdaModel

In [10]:

%time model = LdaModel(corpus=corpus, id2word=id2word, chunksize=chunksize, \alpha='auto', eta='auto', \iterations=iterations, num_topics=num_topics, \passes=passes, eval_every=eval_every)
CPU times: user 3min 58s, sys: 348 ms, total: 3min 58sWall time: 3min 59s

import pyLDAvis.gensimpyLDAvis.enable_notebook()
import warningswarnings.filterwarnings("ignore", category=DeprecationWarning)
In [12]:
pyLDAvis.gensim.prepare(model, corpus, dictionary)

from sklearn.metrics.pairwise import cosine_similarity
p_df['tokenz'] = docs
docs1 = p_df['tokenz'].apply(lambda l: l[:int0(len(l)/2)])docs2 = p_df['tokenz'].apply(lambda l: l[int0(len(l)/2):])

corpus1 = [dictionary.doc2bow(doc) for doc in docs1]corpus2 = [dictionary.doc2bow(doc) for doc in docs2]
# 使用语料库LDA模型转换

lda_corpus1 = model[corpus1]lda_corpus2 = model[corpus2]

from collections import OrderedDictdef get_doc_topic_dist(model, corpus, kwords=False):
'''LDA转换，对于每个文档，仅返回权重非零的主题此函数对主题空间中的文档进行矩阵转换'''top_dist =[]keys = []
for d in corpus:tmp = {i:0 for i in range(num_topics)}tmp.update(dict(model[d]))vals = list(OrderedDict(tmp).values())top_dist += [array(vals)]if kwords:keys += [array(vals).argmax()]
return array(top_dist), keys

Intra similarity: cosine similarity for corresponding parts of a doc(higher is better):0.906086532099Inter similarity: cosine similarity between random parts (lower is better):0.846485334252

def explore_topic(lda_model, topic_number, topn, output=True):"""

输出topn词的列表"""terms = []for term, frequency in lda_model.show_topic(topic_number, topn=topn):terms += [term]if output:print(u'{:20} {:.3f}'.format(term, round(frequency, 3)))
return terms

term                 frequency
Topic 0 |---------------------
data_set             0.006embedding            0.004query                0.004document             0.003tensor               0.003multi_label          0.003graphical_model      0.003singular_value       0.003topic_model          0.003margin               0.003Topic 1 |---------------------
policy               0.007regret               0.007bandit               0.006reward               0.006active_learning      0.005agent                0.005vertex               0.005item                 0.005reward_function      0.005submodular           0.004Topic 2 |---------------------
convolutional        0.005generative_model     0.005variational_inference 0.005recurrent            0.004gaussian_process     0.004fully_connected      0.004recurrent_neural     0.004hidden_unit          0.004deep_learning        0.004hidden_layer         0.004Topic 3 |---------------------
convergence_rate     0.007step_size            0.006matrix_completion    0.006rank_matrix          0.005gradient_descent     0.005regret               0.004sample_complexity    0.004strongly_convex      0.004line_search          0.003sample_size          0.003

top_labels = {0: 'Statistics', 1:'Numerical Analysis', 2:'Online Learning', 3:'Deep Learning'}

In [20]:
'''# 1.删除非字母

paper_text = re.sub("[^a-zA-Z]"," ", paper)# 2.将单词转换为小写并拆分

words = paper_text.lower().split()# 3. 删除停用词

words = [w for w in words if not w in stops]# 4. 删除短词

words = [t for t in words if len(t) > 2]# 5. 形容词

words = [nltk.stem.WordNetLemmatizer().lemmatize(t) for t in words]
In [21]:
from sklearn.feature_extraction.text import TfidfVectorizer
tvectorizer = TfidfVectorizer(input='content', analyzer = 'word', lowercase=True, stop_words='english',\tokenizer=paper_to_wordlist, ngram_range=(1, 3), min_df=40, max_df=0.20,\norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=True)
dtm = tvectorizer.fit_transform(p_df['PaperText']).toarray()
In [22]:
top_dist =[]for d in corpus:tmp = {i:0 for i in range(num_topics)}tmp.update(dict(model[d]))vals = list(OrderedDict(tmp).values())top_dist += [array(vals)]
In [23]:
top_dist, lda_keys= get_doc_topic_dist(model, corpus, True)features = tvectorizer.get_feature_names()
In [24]:
top_ws = []for n in range(len(dtm)):inds = int0(argsort(dtm[n])[::-1][:4])tmp = [features[i] for i in inds]
top_ws += [' '.join(tmp)]


cluster_colors = {0: 'blue', 1: 'green', 2: 'yellow', 3: 'red', 4: 'skyblue', 5:'salmon', 6:'orange', 7:'maroon', 8:'crimson', 9:'black', 10:'gray'}
p_df['colors'] = p_df['clusters'].apply(lambda l: cluster_colors[l])
In [25]:
from sklearn.manifold import TSNEtsne = TSNE(n_components=2)X_tsne = tsne.fit_transform(top_dist)
In [26]:
p_df['X_tsne'] =X_tsne[:, 0]p_df['Y_tsne'] =X_tsne[:, 1]
In [27]:
from bokeh.plotting import figure, show, output_notebook, save#输出文件from bokeh.models import HoverTool, value, LabelSet, Legend, ColumnDataSourceoutput_notebook()
 BokehJS 0.12.5成功加载。
In [28]:
source = ColumnDataSource(dict(x=p_df['X_tsne'],y=p_df['Y_tsne'],color=p_df['colors'],label=p_df['clusters'].apply(lambda l: top_labels[l]),#     msize= p_df['marker_size'],topic_key= p_df['clusters'],title= p_df[u'Title'],content = p_df['Text_Rep']))
In [29]:
title = 'T-SNE visualization of topics'


plot_lda.scatter(x='x', y='y', legend='label', source=source,color='color', alpha=0.8, size=10)#'msize', )
show(plot_lda)