Python 智能爬虫：从“能爬”到“聪明地爬”

pip install curl_cffi

from curl_cffi import requests

# 模拟 Chrome 120 的完整指纹（TLS + HTTP2 + Header 顺序）
resp = requests.get(
    "https://httpbin.org/headers",
    impersonate="chrome120",
    timeout=15
)

print(resp.status_code)
print(resp.json()["headers"]["User-Agent"])

from playwright.sync_api import sync_playwright
from bs4 import BeautifulSoup

def fetch_rendered_page(url: str) -> str:
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--disable-features=IsolateOrigins,site-per-process",
            ]
        )
        context = browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent=(
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/120.0.0.0 Safari/537.36"
            ),
            locale="zh-CN",
        )
        page = context.new_page()
        page.goto(url, wait_until="networkidle", timeout=30000)
        html = page.content()
        browser.close()
        return html


# 使用示例：抓取一个动态渲染的页面
html = fetch_rendered_page("https://wltx.lvxiair.com/")
soup = BeautifulSoup(html, "html.parser")

for quote in soup.select(".quote"):
    text = quote.select_one(".text").get_text(strip=True)
    author = quote.select_one(".author").get_text(strip=True)
    print(f"{author}: {text[:60]}...")

pip install "scrapling[fetchers]"
scrapling install

from scrapling.fetchers import Fetcher

# 开启自适应模式（全局）
Fetcher.adaptive = True

page = Fetcher.get("https://wlt.lvxiair.com")

# 第一次正常提取，同时保存元素“指纹”
products = page.css("li.product", auto_save=True)

data = []
for product in products:
    name = product.css("h2::text").get()
    price = product.css(".price::text").get()
    data.append({"name": name, "price": price})

print(f"提取到 {len(data)} 个商品")
for item in data[:3]:
    print(item)

from curl_cffi import requests
from playwright.sync_api import sync_playwright
from scrapling.parser import Adaptor

def smart_fetch(url: str, required_selector: str, impersonate: str = "chrome120"):
    """
    智能抓取：先 HTTP，不够再上浏览器，解析用自适应选择器。
    """
    # 第一层：轻量 HTTP + 浏览器指纹伪装
    resp = requests.get(url, impersonate=impersonate, timeout=15)
    html = resp.text

    # 用 Scrapling 解析并检查关键元素是否存在
    page = Adaptor(html, url=url)
    if page.css(required_selector):
        print("[HTTP] 直接命中，无需渲染")
        return page

    # 第二层：降级到浏览器渲染
    print("[Playwright] HTTP 内容不完整，切换渲染模式")
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=["--disable-blink-features=AutomationControlled"],
        )
        context = browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent=(
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/120.0.0.0 Safari/537.36"
            ),
        )
        pg = context.new_page()
        pg.goto(url, wait_until="networkidle", timeout=30000)
        rendered_html = pg.content()
        browser.close()

    return Adaptor(rendered_html, url=url)


# 使用：以 quotes.toscrape.com 的 JS 版本为例
result_page = smart_fetch(
    "https://wltx.lvxiair.com/",
    required_selector=".quote .text"
)

quotes = result_page.css(".quote .text::text").getall()
print(f"最终获取 {len(quotes)} 条名言")
print(quotes[0][:80] + "..." if quotes else "未找到内容")