构建混合爬虫：何时使用Requests，何时切换至Selenium处理请求头？

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

def login_with_selenium():
    """使用Selenium模拟登录，并返回关键的Cookies和User-Agent"""
    print("启动Selenium进行登录...")
    
    # 配置浏览器选项（可选：无头模式）
    options = webdriver.ChromeOptions()
    # options.add_argument('--headless') # 生产环境可开启
    options.add_argument('--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
    
    driver = webdriver.Chrome(options=options)
    
    try:
        # 访问登录页面
        driver.get('https://example.com/login')
        
        # 显式等待，找到用户名和密码输入框
        username_input = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.ID, "username"))
        )
        password_input = driver.find_element(By.ID, "password")
        
        # 输入凭据并提交
        username_input.send_keys("your_username")
        password_input.send_keys("your_password")
        driver.find_element(By.ID, "login-btn").click()
        
        # 等待登录成功，例如等待某个登录后才出现的元素
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, "user-dashboard"))
        )
        print("登录成功！")
        
        # 关键步骤：获取登录后的Cookies和浏览器实际的User-Agent
        cookies = driver.get_cookies()
        user_agent = driver.execute_script("return navigator.userAgent;")
        
        return cookies, user_agent
        
    finally:
        # 关闭浏览器
        driver.quit()

# 执行登录函数
selenium_cookies, selenium_user_agent = login_with_selenium()

import requests

def create_requests_session(cookies, user_agent):
    """创建一个携带了Selenium状态的Requests会话"""
    session = requests.Session()
    
    # 1. 设置User-Agent
    session.headers.update({
        'User-Agent': user_agent,
        # 可以在这里添加其他在Selenium中观察到的固定请求头
        'Referer': 'https://example.com/',
        'Accept': 'application/json, text/plain, */*' # 根据实际情况修改
    })
    
    # 2. 处理Cookies：将Selenium的cookies格式转换为Requests可用的格式
    requests_cookies = {}
    for cookie in cookies:
        requests_cookies[cookie['name']] = cookie['value']
        
    # 将cookies更新到会话中
    session.cookies.update(requests_cookies)
    
    return session

# 创建混合爬虫会话
hybrid_session = create_requests_session(selenium_cookies, selenium_user_agent)

dimport requests

# 代理配置信息
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

def create_requests_session_with_proxy(cookies, user_agent):
    """创建一个携带了Selenium状态和代理配置的Requests会话"""
    session = requests.Session()
    
    # 设置代理
    proxies = {
        'http': f'http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}',
        'https': f'http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}'
    }
    session.proxies.update(proxies)
    
    # 设置User-Agent和其他请求头
    session.headers.update({
        'User-Agent': user_agent,
        'Referer': 'https://example.com/',
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive'
    })
    
    # 处理Cookies
    requests_cookies = {}
    for cookie in cookies:
        requests_cookies[cookie['name']] = cookie['value']
    session.cookies.update(requests_cookies)
    
    return session

# 使用带代理的Session创建函数
hybrid_session_with_proxy = create_requests_session_with_proxy(selenium_cookies, selenium_user_agent)

def fetch_data_with_proxy_session(session):
    """使用已经配置好代理的会话请求数据"""
    api_url = "https://example.com/api/data-list"
    
    try:
        # 直接使用session，代理已经在session中配置好了
        response = session.get(api_url, timeout=30)
        response.raise_for_status()
        
        data = response.json()
        print(f"成功获取到 {len(data.get('items', []))} 条数据。")
        
        for item in data['items']:
            print(f"处理数据: {item['id']} - {item['title']}")
            
        return data
        
    except requests.exceptions.RequestException as e:
        print(f"请求数据时发生错误: {e}")
        return None

# 执行数据抓取
data = fetch_data_with_proxy_session(hybrid_session_with_proxy)