用深度学习提升DOM解析——自动提取页面关键区块

import asyncio
from playwright.async_api import async_playwright
import base64
import json
import torch
from transformers import BertTokenizer, BertForTokenClassification
from typing import List

# === 亿牛云代理配置 ===
PROXY_SERVER = "http://proxy.16yun.cn:3100"
PROXY_USERNAME = "16YUN"
PROXY_PASSWORD = "16IP"

# Base64编码代理身份
def get_proxy_auth():
    credentials = f"{PROXY_USERNAME}:{PROXY_PASSWORD}"
    return "Basic " + base64.b64encode(credentials.encode()).decode()

# === 深度学习模型加载（模拟）===
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
model = BertForTokenClassification.from_pretrained("bert-base-chinese")

def predict_blocks(text_list: List[str]) -> List[str]:
    """
    使用BERT模拟DOM区块分类
    输入：文本列表（如标签+内容组成）
    输出：每个文本的分类结果，如[车型] [年限] [报价]
    """
    inputs = tokenizer(text_list, return_tensors="pt", padding=True, truncation=True)
    with torch.no_grad():
        outputs = model(**inputs).logits
    predictions = torch.argmax(outputs, dim=-1)
    labels = [model.config.id2label[idx.item()] for idx in predictions[0]]
    return labels

# === Playwright爬虫主流程 ===
async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)

        # 启动代理 + 设置UA和Cookie
        context = await browser.new_context(
            proxy={"server": PROXY_SERVER},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122.0.0.0 Safari/537.36",
            extra_http_headers={
                "Proxy-Authorization": get_proxy_auth(),
                "Cookie": "your_cookie_here"  # 真实抓包填入cookie
            }
        )
        
        page = await context.new_page()

        # 搜索“北京二手车”
        await page.goto("https://www.dongchedi.com/usedcar/search?city_name=%E5%8C%97%E4%BA%AC")
        await page.wait_for_timeout(5000)  # 等待JS加载完

        # 提取页面HTML
        html = await page.content()

        # 简化提取所有车卡片文本（真实项目中解析DOM结构）
        elements = await page.query_selector_all("div.CarCard_card__...")  # 选择器根据真实页面调整
        for el in elements:
            text = await el.inner_text()
            label = predict_blocks([text])[0]
            print(f"[{label}] {text}")

        await browser.close()

asyncio.run(main())

[静态爬虫Requests+XPath] ──> [动态爬虫Playwright+XPath] ──> [Playwright+DOM语义BERT]
             ❌失效                          ⚠️结构脆弱                     ✅语义提取，稳定高效