Python爬虫实战：如何优雅地处理超时和延迟加载问题

import requests

url = "https://example.com"
try:
    # 设置连接超时（connect timeout）和读取超时（read timeout）
    response = requests.get(url, timeout=(3, 10))  # 3秒连接超时，10秒读取超时
    print(response.status_code)
except requests.exceptions.Timeout:
    print("请求超时，请检查网络或目标服务器状态")
except requests.exceptions.RequestException as e:
    print(f"请求失败: {e}")

import aiohttp
import asyncio

async def fetch(session, url):
    try:
        async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as response:
            return await response.text()
    except asyncio.TimeoutError:
        print("异步请求超时")
    except Exception as e:
        print(f"请求失败: {e}")

async def main():
    async with aiohttp.ClientSession() as session:
        html = await fetch(session, "https://example.com")
        print(html[:100])  # 打印前100字符

asyncio.run(main())

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time

driver = webdriver.Chrome()
driver.get("https://example.com/lazy-load-page")

# 模拟滚动到底部，触发加载
for _ in range(3):  # 滚动3次
    driver.find_element(By.TAG_NAME, 'body').send_keys(Keys.END)
    time.sleep(2)  # 等待数据加载

# 获取完整页面
full_html = driver.page_source
print(full_html)

driver.quit()

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    page.goto("https://example.com/lazy-load-page")

    # 模拟滚动
    for _ in range(3):
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(2000)  # 等待2秒

    # 获取完整HTML
    full_html = page.content()
    print(full_html[:500])  # 打印前500字符

    browser.close()

import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
def fetch_with_requests(url):
    try:
        response = requests.get(url, timeout=(3, 10))
        return response.text
    except requests.exceptions.Timeout:
        print("请求超时，尝试使用Selenium")
        return None
def fetch_with_selenium(url):
    driver = webdriver.Chrome()
    driver.get(url)
    # 模拟滚动3次
    for _ in range(3):
        driver.find_element(By.TAG_NAME, 'body').send_keys(Keys.END)
        time.sleep(2)
    html = driver.page_source
    driver.quit()
    return html
def main():
    url = "https://example-shop.com/products"
    # 先尝试用requests（更快）
    html = fetch_with_requests(url)
    # 如果失败，改用Selenium（处理动态加载）
    if html is None or "Loading more products..." in html:
        html = fetch_with_selenium(url)
    # 解析数据（示例：提取商品名称）
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    products = soup.find_all('div', class_='product-name')
    for product in products[:10]:  # 打印前10个商品
        print(product.text.strip())
if __name__ == "__main__":
    main()