手把手教你用Python分析电影 | 以《蚁人2》为例

# coding=utf-8
from urllib import request
import json
import time
from datetime import datetime
from datetime import timedelta
import random
# 获取数据，根据url获取
def get_data(url):
    headers = {
        ‘User-Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36’
    }
    req = request.Request(url, headers=headers)
    response = request.urlopen(req)
    if response.getcode() == 200:
        return response.read()
    return None

if __name__ == ‘__main__’:
    html = get_data(‘http://m.maoyan.com/mmdb/comments/movie/343208.json?_v_=yes&offset=0&startTime=2018-09-20%2022%3A25%3A03’)
    print(html)

def parse_data(html):
    data = json.loads(html)[‘cmts’]  # 将str转换为json
    comments = []
    for item in data:
        comment = {
            ‘id’: item[‘id’],
            ‘nickName’: item[‘nickName’],
            ‘cityName’: item[‘cityName’] if ‘cityName’ in item else ‘’,  # 处理cityName不存在的情况
            ‘content’: item[‘content’].replace(‘\n’, ‘ ‘, 10),  # 处理评论内容换行的情况
            ‘score’: item[‘score’],
            ‘startTime’: item[‘startTime’]
        }
        comments.append(comment)
    return comments

if __name__ == ‘__main__’:
    html = get_data(‘http://m.maoyan.com/mmdb/comments/movie/343208.json?_v_=yes&offset=0&startTime=2018-09-20%2022%3A25%3A03’)
    comments = parse_data(html)
    print(comments)

# 存储数据，存储到文本文件
def save_to_txt():
    start_time = ‘2018-09-20 22:00:0’ 
    end_time = ‘2018-08-23 00:00:00’

    while start_time > end_time:
        url = ‘http://m.maoyan.com/mmdb/comments/movie/343208.json?_v_=yes&offset=0&startTime=‘ + start_time.replace(‘ ‘, ‘%20’)
        html = None

        try:
            html = get_data(url)
        except Exception as e:
            time.sleep(1 + float(random.randint(1, 50)) / 20)
            html = get_data(url)
        else:
            time.sleep(0.8)

        comments = parse_data(html)
        print(comments)
        start_time = comments[14][‘startTime’]  # 获得末尾评论的时间
        start_time = datetime.strptime(start_time, ‘%Y-%m-%d %H:%M:%S’) + timedelta(seconds=-1)  # 转换为datetime类型，减1秒，避免获取到重复数据
        start_time = datetime.strftime(start_time, ‘%Y-%m-%d %H:%M:%S’)  # 转换为str

        for item in comments:
            with open(‘ant-man2.txt’, ‘a’, encoding=‘utf-8’) as f:
                f.write(str(item[‘id’])+’,’+item[‘nickName’] + ‘,’ + item[‘cityName’] + ‘,’ + item[‘content’] + ‘,’ + str(item[‘score’])+ ‘,’ + item[‘startTime’] + ‘\n’)

if __name__ == ‘__main__’:
    html = get_data(‘http://m.maoyan.com/mmdb/comments/movie/343208.json?_v_=yes&offset=0&startTime=2018-09-20%2022%3A25%3A03’)
    comments = parse_data(html)
    print(comments)
    save_to_txt()

count = 0
for index, line in enumerate(open(r’C:\Users\Administrator\.PyCharmCE2018.2\config\scratches\ant-man2.txt’,encoding=‘utf-8’)):
    count += 1
print(count)

# coding=utf-8
# 导入Style类，用于定义样式风格
from pyecharts import Style
# 导入Geo组件，用于生成地理坐标类图
from pyecharts import Geo
import json
# 导入Geo组件，用于生成柱状图
from pyecharts import Bar
# 导入Counter类，用于统计值出现的次数
from collections import Counter
# 处理地名数据，解决坐标文件中找不到地名的问题
def handle(cities):
    # print(len(cities), len(set(cities)))
    # 获取坐标文件中所有地名
    data = None
    with open(
‘C:/Users/Administrator/PycharmProjects/zhilianzhaopin/venv/lib/site-packages/pyecharts/datasets/city_coordinates.json’,
            mode=‘r’, encoding=‘utf-8’) as f:
        data = json.loads(f.read())  # 将str转换为json

    # 循环判断处理
    data_new = data.copy()  # 拷贝所有地名数据
    for city in set(cities):  # 使用set去重
        # 处理地名为空的数据
        if city == ‘’:
            while city in cities:
                cities.remove(city)
        count = 0
        for k in data.keys():
            count += 1
            if k == city:
                break
            if k.startswith(city):  # 处理简写的地名，如 达州市 简写为 达州
                # print(k, city)
                data_new[city] = data[k]
                break
            if k.startswith(city[0:-1]) and len(city) >= 3:  # 处理行政变更的地名，如县改区 或 县改市等
                data_new[city] = data[k]
                break
        # 处理不存在的地名
        if count == len(data):
            while city in cities:
                cities.remove(city)

    # 写入覆盖坐标文件
    with open(
‘C:/Users/Administrator/PycharmProjects/zhilianzhaopin/venv/lib/site-packages/pyecharts/datasets/city_coordinates.json’,
            mode=‘w’, encoding=‘utf-8’) as f:
        f.write(json.dumps(data_new, ensure_ascii=False))  # 将json转换为str

# 数据可视化
def render():
    # 获取评论中所有城市
    cities = []
    with open(‘ant-man2.txt’, mode=‘r’, encoding=‘utf-8’) as f:
        rows = f.readlines()
        for row in rows:
            print(row)
            city = row.split(‘,’)[2]

            If city != ‘’:  # 去掉城市名为空的值
                cities.append(city)

    # 定义样式
    style = Style(
        title_color=‘#fff',
        title_pos='c‘nter',’        width=1200,
        height=600,
        background_color='‘404a59'
 ’  )

    # 根据城市数据生成地理坐标图
    geoGeoGeo('《蚁‘2》粉丝位置分布', ’数据‘源：猫眼电影-老司机采集', ’*style.init_style)
    attr, value = geo.cast(data)
    geo.add('',‘’ttr, value, visual_range=[0, 3500],
            visual_text_color='#‘ff', symbol_size=15,
            is_visualmap=True, is_piecewise=True, visual_split_number=10)
    geo.render('粉丝‘置分布-地理坐标图.html')

# 根据城市数据生成柱状图
    data_top20 = Counter(cities).most_common(20)  # 返回出现次数最多的20条
    bar = Bar('《蚁‘2》粉丝来源排行TOP20', ’数据‘源：猫眼电影-老司机采集', ’itle_pos='ce‘ter', ’idth=1200, height=600)
    attr, value = bar.cast(data_top20)
    bar.add('',‘attr, value, is_visualmap=True, visual_range=[0, 3500], visual_text_color='#fff', is_more_utils=True,
            is_label_show=True)
    bar.render('粉丝‘源排行-柱状图.html')
’if __name__ == '__‘ain__':
’   render()

# coding=utf-8
# 导入Pie组件，用于生成饼图
from pyecharts import Pie

# 获取评论中所有评分
rates = []
with open('an‘-man2.txt', ’ode='r'‘ ’ncoding='ut‘-8') ’s f:
    rows = f.readlines()
    for row in rows:
        try:
            rate = row.split(','‘[’]
        except IndexError:  #遇到报错则跳过
            continue
        if rate!= '':‘’           rates.append(rate)

# print(rates)

# 定义星级，并统计各星级评分数量
attAttr['五星‘, ’四星‘, ’三星‘, ’二星‘, ’一星‘]
’alue = [
    rates.count('5'‘ ’ rates.count('4.‘'),’    rates.count('4'‘ ’ rates.count('3.‘'),’    rates.count('3'‘ ’ rates.count('2.‘'),’    rates.count('2'‘ ’ rates.count('1.‘'),’    rates.count('1'‘ ’ rates.count('0.‘')
’
# print(value)

pie = Pie('《蚁‘2》评分星级比例', ’itle_pos='ce‘ter', ’idth=900)
pie.add('7-‘7', ’ttr, value, center=[75, 50], is_random=True,
        radius=[30, 75], rosetype='ar‘a',
’       is_legend_show=False, is_label_show=True)
pie.render('《蚁‘2》评分.html')

# coding=utf-8
import jieba
import matplotlib.pyplot as plt

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

# 获取所有评论
comments = []
with open('an‘-man2.txt', ’ode='r'‘ ’ncoding='ut‘-8') ’s f:
    rows = f.readlines()
    for row in rows:
        try:
            comment = row.split(','‘[’]
        except IndexError:       #跳过报错
            continue
        if comment != '':‘’           comments.append(comment)

print(comments)

# 设置分词
comment_after_split = jieba.cut(str(comments), cut_all=False)  # 非全模式分词，cut_all=false
words = ' '‘j‘in(comment_after_split)  # 以空格进行拼接
print(words)

# 设置屏蔽词
stopwords = STOPWORDS.copy()
stopwords.add('电影‘)
’topwords.add('一部‘)
’topwords.add('一个‘)
’topwords.add('没有‘)
’topwords.add('什么‘)
’topwords.add('有点‘)
’topwords.add('这部‘)
’topwords.add('这个‘)
’topwords.add('不是‘)
’topwords.add('真的‘)
’topwords.add('感觉‘)
’topwords.add('觉得‘)
’topwords.add('还是‘)
’topwords.add('但是‘)
’topwords.add('就是‘)
’topwords.add('蚁人‘)
’# 导入背景图
bg_image = plt.imread('an‘-man2.jpg')
’# 设置词云参数，参数分别表示：画布宽高、背景颜色、背景图形状、字体、屏蔽词、最大词的字体大小
wc = WordCloud(width=2024, height=1400, background_color='wh‘te', ’ask=bg_image, font_path='ST‘AITI.TTF',
’              stopwords=stopwords, max_font_size=400, random_state=50)

# 将分词后数据传入云图
wc.generate_from_text(words)
plt.imshow(wc)
plt.axis('of‘') ’# 不显示坐标轴
plt.show()

# 保存结果到本地
wc.to_file('蚁人‘词云图2.jpg')