构建基于内容的数据科学文章推荐器

import numpy as np
import pandas as pd
import re
import string
from sklearn.decomposition import NMF
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
import gensim
from gensim.parsing.preprocessing import STOPWORDS
from gensim import corpora, models
from gensim.utils import simple_preprocess
from nltk.stem.porter import PorterStemmer
medium = pd.read_csv(‘Medium_AggregatedData.csv’)
medium.head()

# Filter articles
medium = medium[medium['language'] == 'en']
medium = medium[medium['totalClapCount'] >= 25]
def findTags(title):
    '''
    Function extracts tags for an input title
    '''
    rows = medium[medium['title'] == title]
    tags = list(rows['tag_name'].values)
    return tags
# Get all the titles
titles = medium['title'].unique()
tag_dict = {'title': [], 'tags': []} # Dictionary to store tags
for title in titles:
    tag_dict['title'].append(title)
    tag_dict['tags'].append(findTags(title))
tag_df = pd.DataFrame(tag_dict)  # Dictionary to data frame
# Now that tag data is extracted the duplicate rows can be dropped
medium = medium.drop_duplicates(subset = 'title', keep = 'first')
def addTags(title):
    '''
    Adds tags back into medium data frame as a list
    '''
    try:
        tags = list(tag_df[tag_df['title'] == title]['tags'])[0]
    except:
        # If there's an error assume no tags
        tags = np.NaN
    return tags
# Apply addTags
medium['allTags'] = medium['title'].apply(addTags)
# Keep only the columns we're interested in for this project
keep_cols = ['title', 'url', 'allTags', 'readingTime',
             'author', 'text']
medium = medium[keep_cols]
# Drop row with null title
null_title = medium[medium['title'].isna()].index
medium.drop(index = null_title, inplace = True)
medium.reset_index(drop = True, inplace = True)
print(medium.shape)
medium.head()

def clean_text(text):
    '''
    Eliminates links, non alphanumerics, and punctuation.
    Returns lower case text.
    '''
    
    # Remove links
    text = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+
                  \.[\w/\-?=%.]+','', text)
    # Remove non-alphanumerics
    text = re.sub('\w*\d\w*', ' ', text)
    # Remove punctuation and lowercase
    text = re.sub('[%s]' % re.escape(string.punctuation),
                  ' ', text.lower())
    # Remove newline characters
    text = text.replace('\n', ' ')
    
    return text
medium['text'] = medium['text'].apply(clean_text)

stop_list = STOPWORDS.union(set(['data', 'ai', 'learning', 'time', 'machine', 'like', 'use', 'new', 'intelligence', 'need', "it's", 'way', 'artificial', 'based', 'want', 'know', 'learn', "don't", 'things', 'lot', "let's", 'model', 'input', 'output', 'train', 'training', 'trained', 'it', 'we', 'don', 'you', 'ce', 'hasn', 'sa', 'do', 'som', 'can']))
# Remove stopwords
def remove_stopwords(text):
    clean_text = []
    for word in text.split(' '):
        if word not in stop_list and (len(word) > 2):
            clean_text.append(word)
    return ' '.join(clean_text)
medium['text'] = medium['text'].apply(remove_stopwords)

# Apply stemmer to processedText
stemmer = PorterStemmer()
def stem_text(text):
    word_list = []
    for word in text.split(' '):
        word_list.append(stemmer.stem(word))
    return ' '.join(word_list)
medium['text'] = medium['text'].apply(stem_text)

medium.to_csv('pre-processed.csv')

vectorizer = TfidfVectorizer(stop_words = stop_list,
                             ngram_range = (1,1))
doc_word = vectorizer.fit_transform(medium['text'])
svd = TruncatedSVD(8)
docs_svd = svd.fit_transform(doc_word)

def display_topics(model, feature_names, no_top_words, no_top_topics, topic_names=None):
    count = 0
    for ix, topic in enumerate(model.components_):
        if count == no_top_topics:
            break
        if not topic_names or not topic_names[ix]:
            print("\nTopic ", (ix + 1))
        else:
            print("\nTopic: '",topic_names[ix],"'")
        print(", ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words -
                                                 1:-1]]))
        count += 1
display_topics(svd, vectorizer.get_feature_names(), 15, 8)

nmf = NMF(8)
docs_nmf = nmf.fit_transform(doc_word)
display_topics(nmf, vectorizer.get_feature_names(), 15, 8)

tokenized_docs = medium['text'].apply(simple_preprocess)
dictionary = gensim.corpora.Dictionary(tokenized_docs)
dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)
corpus = [dictionary.doc2bow(doc) for doc in tokenized_docs]
# Workers = 4 activates all four cores of my CPU,
lda = models.LdaMulticore(corpus=corpus, num_topics=8,
                          id2word=dictionary, passes=10,
                          workers = 4)
lda.print_topics()

# Define column names for dataframe
column_names = ['title', 'url', 'allTags', 'readingTime', 'author',
                'Tech', 'Modeling', 'Chatbots', 'Deep Learning',
                'Coding', 'Business', 'Careers', 'NLP', 'sum']
# Create topic sum for each article
# Later remove all articles with sum 0
topic_sum = pd.DataFrame(np.sum(docs_nmf, axis = 1))
# Turn our docs_nmf array into a data frame
doc_topic_df = pd.DataFrame(data = docs_nmf)
# Merge all of our article metadata and name columns
doc_topic_df = pd.concat([medium[['title', 'url', 'allTags',
                         'readingTime', 'author']], doc_topic_df,
                         topic_sum], axis = 1)
doc_topic_df.columns = column_names
# Remove articles with topic sum = 0, then drop sum column
doc_topic_df = doc_topic_df[doc_topic_df['sum'] != 0]
doc_topic_df.drop(columns = 'sum', inplace = True)
# Reset index then save
doc_topic_df.reset_index(drop = True, inplace = True)
doc_topic_df.to_csv('tfidf_nmf_8topics.csv', index = False)

topic_names = ['Tech', 'Modeling', 'Chatbots', 'Deep Learning',
               'Coding', 'Business', 'Careers', 'NLP']
topic_array = np.array(doc_topic_df[topic_names])
norms = np.linalg.norm(topic_array, axis = 1)
def compute_dists(top_vec, topic_array):
    '''
    Returns cosine distances for top_vec compared to every article
    '''
    dots = np.matmul(topic_array, top_vec)
    input_norm = np.linalg.norm(top_vec)
    co_dists = dots / (input_norm * norms)
    return co_dists
def produce_rec(top_vec, topic_array, doc_topic_df, rand = 15):
    '''
    Produces a recommendation based on cosine distance.
    rand controls magnitude of randomness.
    '''
    top_vec = top_vec + np.random.rand(8,)/
                        (np.linalg.norm(top_vec)) * rand
    co_dists = compute_dists(top_vec, topic_array)
return doc_topic_df.loc[np.argmax(co_dists)]

tech = 5
modeling = 5
chatbots = 0
deep = 0
coding = 0
business = 5
careers = 0
nlp = 0
top_vec = np.array([tech, modeling, chatbots, deep, coding, business, careers, nlp])
rec = produce_rec(top_vec, topic_array, doc_topic_df)
Rec