数据可视化|用Python实现手机抓包，获取当当图书差评数据！

import json

def response(flow):
    url = 'product.mapi.dangdang.com'
    page_size = 'page_size=15'
    # 对url进行筛选,只选取评论的url
    if url and page_size in flow.request.url:
        text = flow.response.text
        data = json.loads(text)
        for item in data['review_list']:
            # 获取用户昵称
            if len(item['cust_name']) > 0:
                name = item['cust_name']
            else:
                name = '无名'
            print(item['cust_name'])
            # 获取用户评分
            if len(item['score']) > 0:
                score = str(item['score'])
            else:
                score = '0'
            print(item['score'] + '\n')
            # 获取用户评论
            content = item['content'].replace(',', '，').replace('\n', '')
            print(item['content'] + '\n')
            # 获取用户评论时间
            creation_date = item['creation_date']
            print(item['creation_date'])
            # 获取有用数
            if len(str(item['total_helpful_num'])) > 0 :
                total_helpful_num = str(item['total_helpful_num'])
            else:
                total_helpful_num = '0'
            print(item['total_helpful_num'])
            # 获取无用数
            if len(str(item['total_useless_num'])) > 0 :
                total_useless_num = str(item['total_useless_num'])
            else:
                total_useless_num = '0'
            print(item['total_useless_num'])
            # 获取评论数
            if len(str(item['total_reply_num'])) > 0 :
                total_reply_num = str(item['total_reply_num'])
            else:
                total_reply_num = '0'
            print(item['total_reply_num'])
            print('\n')
            # 将获取信息写入csv文件
            with open('alive.csv', 'a+', encoding='utf-8-sig') as f:
                f.write(name + ',' + score + ',' + content + ',' + creation_date + ',' + total_helpful_num + ',' + total_useless_num + ',' + total_reply_num + '\n')

import time
import random
from appium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from appium.webdriver.common.touch_action import TouchAction
from selenium.webdriver.support import expected_conditions as EC


def main():
    # 设置驱动配置
    server = 'http://localhost:4723/wd/hub'
    desired_caps = {
        'platformName': 'Android',
        'deviceName': 'STF_AL00',
        'appPackage': 'com.dangdang.buy2',
        'appActivity': 'com.dangdang.buy2.StartupActivity'
    }
    driver = webdriver.Remote(server, desired_caps)
    # 这里获取一下手机屏幕实际大小,可以为设置滑动参数做参考
    size = driver.get_window_size()
    print(size)
    wait = WebDriverWait(driver, 60)
    # 因为要叫我切换地区，选择取消
    button_1 = wait.until(EC.presence_of_element_located((By.ID, 'com.dangdang.buy2:id/left_bt')))
    button_1.click()
    # 点击图书榜按钮
    button_2 = wait.until(EC.presence_of_element_located((By.ID, 'com.dangdang.buy2:id/index_icon_iv0')))
    button_2.click()
    # 点击图书「活着」区域块
    button_3 = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.FrameLayout/android.widget.RelativeLayout/android.support.v4.view.ViewPager/android.widget.RelativeLayout/android.widget.ListView/android.widget.LinearLayout[2]')))
    button_3.click()
    # 点击评论区域块
    button_4 = wait.until(EC.presence_of_element_located((By.ID, 'com.dangdang.buy2:id/product_component_book_score_ll')))
    button_4.click()
    time.sleep(5)
    # 点击差评按钮
    button_5 = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/android.widget.LinearLayout/android.widget.FrameLayout/android.widget.RelativeLayout/android.support.v4.view.ViewPager/android.widget.LinearLayout/android.support.v4.view.ViewPager/android.widget.RelativeLayout/android.widget.LinearLayout/android.view.ViewGroup/android.widget.RelativeLayout[6]/android.widget.TextView')))
    button_5.click()
    # 向下滑动,y轴参数我随便选的,向上滑就对了
    while True:
        TouchAction(driver).press(x=515, y=1247).move_to(x=515, y=1026).release().perform()
        time.sleep(float(random.randint(5, 10)))


if __name__ == '__main__':
    main()

from wordcloud import WordCloud, ImageColorGenerator
import matplotlib.pyplot as plt
import pandas as pd
import jieba

df = pd.read_excel('alive.xlsx', header=None, names=["name", "score", "comment", "date", "up_number", "down_number", "reply_number"])

text = ''
for line in df['comment']:
    text += ' '.join(jieba.cut(line, cut_all=False))
backgroud_Image = plt.imread('book.jpg')
stopwords = set('')
stopwords.update(['没有', '什么', '不是', '知道', '怎么', '就是', '本书', '当当', '这个 商品', '一个', '自己', '真的', '商品 不太好', '一本', '这样', '但是', '现在', '你们', '一直', '以后', '这个', '商品'])

wc = WordCloud(
    background_color='white',
    mask=backgroud_Image,
    font_path='C:\Windows\Fonts\STZHONGS.TTF',
    max_words=2000,
    max_font_size=150,
    random_state=30,
    stopwords=stopwords
)
wc.generate_from_text(text)
# 看看词频高的有哪些,把无用信息去除
process_word = WordCloud.process_text(wc, text)
sort = sorted(process_word.items(), key=lambda e:e[1], reverse=True)
print(sort[:50])
img_colors = ImageColorGenerator(backgroud_Image)
wc.recolor(color_func=img_colors)
plt.imshow(wc)
plt.axis('off')
wc.to_file("活着.jpg")
print('生成词云成功!')