使用Python进行网站数据爬取和视频处理

# 导入所需库
import requests
import re
import os
import threading
from moviepy.editor import *

# 亿牛云 爬虫代理加强版 代理服务器信息
proxyHost = "www.16yun.cn"
proxyPort = "3111"
proxyUser = "16YUN"
proxyPass = "16IP"

# 构建代理字典
proxies = {
    "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    "https": f"https://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
}

# 设置请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36"
}

# 定义视频文件保存路径和处理后视频路径
video_path = "videos"
output_path = "output"

# 下载视频函数
def download_video(video_url, filename):
    video_data = requests.get(video_url, headers=headers, proxies=proxies).content
    video_file = os.path.join(video_path, filename)
    with open(video_file, "wb") as f:
        f.write(video_data)
    print(f"下载 {filename} 完成")

# 处理视频函数
def process_video(video_name):
    video_file = os.path.join(video_path, video_name)
    # 使用VideoFileClip方法，读取视频文件并进行剪辑，只保留前10秒
    clip = VideoFileClip(video_file).subclip(0, 10)
    return clip

# 主函数
def main():
    # 定义B站视频网址
    url = "https://www.bilibili.com/video/BV1Xy4y1x7aC"
    # 发送GET请求，获取网页源代码
    response = requests.get(url, headers=headers, proxies=proxies)
    
    # 判断请求是否成功
    if response.status_code == 200:
        print("请求成功")
        html = response.text
        # 使用正则表达式匹配视频链接
        pattern = re.compile(r'"baseUrl":"(.*?)"')
        video_urls = pattern.findall(html)
        
        # 创建视频文件保存路径
        if not os.path.exists(video_path):
            os.mkdir(video_path)
        
        threads = []
        # 遍历视频链接列表，使用多线程下载视频
        for i, video_url in enumerate(video_urls):
            video_name = f"{i+1}.mp4"
            thread = threading.Thread(target=download_video, args=(video_url, video_name))
            threads.append(thread)
            thread.start()
        
        # 等待所有线程完成
        for thread in threads:
            thread.join()
        
        # 创建处理后视频文件保存路径
        if not os.path.exists(output_path):
            os.mkdir(output_path)
        
        clips = []
        # 遍历视频链接列表，处理视频并添加到剪辑列表
        for i in range(len(video_urls)):
            video_name = f"{i+1}.mp4"
            clip = process_video(video_name)
            clips.append(clip)
        
        # 合并剪辑列表中的视频并写入输出文件
        output_clip = concatenate_videoclips(clips)
        output_name = "output.mp4"
        output_file = os.path.join(output_path, output_name)
        output_clip.write_videofile(output_file)
        
        print("处理完成")
    else:
        print("请求失败")

# 确保在主程序中运行
if __name__ == "__main__":
    main()