爬取B站评论：Python技术实现详解

import requests
import json
import os
import pickle
from bs4 import BeautifulSoup
import time

# 设置请求头部信息，伪装成浏览器访问
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}

# 设置代理信息
proxyHost = "xxxx"
proxyPort = "5445"
proxyUser = "xxxxx"
proxyPass = "280651"
proxyMeta = f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"

# 登录B站后获取的cookies，用于自动登录
cookies_file = 'cookies.pkl'

# 保存进度的文件名
progress_file = 'progress.txt'

# 保存评论的文件夹名
comment_dir = 'comments'

# 创建保存评论的文件夹
if not os.path.exists(comment_dir):
    os.makedirs(comment_dir)

# 加载cookies
if os.path.exists(cookies_file):
    with open(cookies_file, 'rb') as f:
        cookies = pickle.load(f)
else:
    cookies = None


def login():
    """手动登录B站，获取cookies"""
    print("请手动登录B站，并复制cookies到cookies.pkl文件中。")


def get_video_id(url):
    """从视频网址中提取视频ID"""
    return url.split('/')[-1]


def get_comments(video_url):
    """爬取视频的评论"""
    video_id = get_video_id(video_url)
    comment_file = os.path.join(comment_dir, f'{video_id}.csv')
    if os.path.exists(comment_file):
        print(f"评论文件 {comment_file} 已存在，跳过该视频。")
        return

    # 请求视频页面，获取评论接口
    response = requests.get(video_url, headers=headers, cookies=cookies, proxies={"http": proxyMeta, "https": proxyMeta})
    soup = BeautifulSoup(response.text, 'html.parser')
    script = soup.find('script', attrs={'type': 'application/ld+json'})
    video_data = json.loads(script.text)
    api_url = video_data['comment']['embedUrl']

    # 循环获取评论，直到获取完所有评论
    page = 1
    comments = []
    while True:
        api = f'{api_url}&pn={page}&type=1'
        response = requests.get(api, headers=headers, cookies=cookies, proxies={"http": proxyMeta, "https": proxyMeta})
        data = response.json()
        if 'data' in data and data['data']['replies']:
            comments.extend(data['data']['replies'])
            page += 1
            time.sleep(1)  # 避免请求过于频繁被封IP
        else:
            break

    # 保存评论到CSV文件
    with open(comment_file, 'w', encoding='utf-8') as f:
        f.write('一级评论计数,隶属关系,被评论者昵称,被评论者ID,评论者昵称,评论者用户ID,评论内容,发布时间,点赞数\n')
        for comment in comments:
            content = comment['content']['message']
            content = content.replace('\n', ' ')
            like = comment['like']
            publish_time = comment['ctime']
            f.write(f'1, , , , , ,"{content}",{publish_time},{like}\n')
            if 'replies' in comment:
                for reply in comment['replies']:
                    content = reply['content']['message']
                    content = content.replace('\n', ' ')
                    like = reply['like']
                    publish_time = reply['ctime']
                    f.write(f'2,{comment["mid"]},{reply["member"]["uname"]},{reply["member"]["mid"]},'
                            f'{reply["member"]["uname"]},{reply["member"]["mid"]},"{content}",{publish_time},{like}\n')
    print(f"成功爬取视频 {video_id} 的评论，保存在 {comment_file} 中。")


def main():
    # 读取视频列表
    with open('video_list.txt', 'r') as f:
        video_urls = f.readlines()

    # 批量爬取视频评论
    for url in video_urls:
        url = url.strip()
        get_comments(url)


if __name__ == '__main__':
    if cookies is None:
        login()
    main()