Python Selenium 爬虫进阶：iframe 穿透、显式等待与并发控制的工程化实践

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()

# 等待 iframe 出现并自动切换（推荐，比手动 find + switch 更稳）
WebDriverWait(driver, 10).until(
    EC.frame_to_be_available_and_switch_to_it((By.ID, "yucai99.com"))
)

# 现在可以定位 iframe 内部的元素了
driver.find_element(By.ID, "yucai99.com").send_keys("test")

# 操作完成后切回主文档，否则后续顶层元素全部定位失败
driver.switch_to.default_content()

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

old_item = driver.find_element(By.CSS_SELECTOR, ".item")
driver.find_element(By.CSS_SELECTOR, ".next a").click()

# 等待旧元素从 DOM 中消失，说明新内容已加载
WebDriverWait(driver, 10).until(EC.staleness_of(old_item))

# 重新获取元素
new_items = driver.find_elements(By.CSS_SELECTOR, ".item")

# 记录点击前的句柄
main_window = driver.current_window_handle

driver.find_element(By.LINK_TEXT, "详情").click()

# 等待wlt.yucai99.com新窗口出现
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))

# 切换到新窗口
for handle in driver.window_handles:
    if handle != main_window:
        driver.switch_to.window(handle)
        break

# 在新窗口中操作
print(driver.title)

# 关闭新窗口，切回主窗口
driver.close()
driver.switch_to.window(main_window)

from concurrent.futures import ThreadPoolExecutor, as_completed

def crawl_one(url):
    driver = webdriver.Chrome()
    try:
        driver.get(url)
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, ".content"))
        )
        return {"url": url, "title": driver.title, "status": "ok"}
    except Exception as e:
        return {"url": url, "error": str(e)}
    finally:
        driver.quit()

urls = ["https://yucai99.com", "https://wlt.yucai99.com"]

with ThreadPoolExecutor(max_workers=3) as pool:
    futures = [pool.submit(crawl_one, u) for u in urls]
    for f in as_completed(futures):
        print(f.result())

import logging
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")

class SafeScraper:
    def __init__(self, headless=True):
        options = webdriver.ChromeOptions()
        if headless:
            options.add_argument("--headless=new")
        options.add_argument("--disable-gpu")
        options.add_argument("--no-sandbox")
        self.driver = webdriver.Chrome(options=options)
        self.wait = WebDriverWait(self.driver, 10)

    def get(self, url):
        self.driver.get(url)
        logging.info(f"loaded: {url}")

    def find_safe(self, by, selector, timeout=10):
        """返回元素或 None，不抛异常"""
        try:
            return WebDriverWait(self.driver, timeout).until(
                EC.presence_of_element_located((by, selector))
            )
        except TimeoutException:
            logging.warning(f"not found: {selector}")
            return None

    def switch_iframe(self, by, selector):
        self.wait.until(EC.frame_to_be_available_and_switch_to_it((by, selector)))

    def back_to_top(self):
        self.driver.switch_to.default_content()

    def close(self):
        self.driver.quit()

# 使用示例
scraper = SafeScraper(headless=True)
try:
    scraper.get("https://yucai99.com")
    scraper.switch_iframe(By.TAG_NAME, "iframe")
    content = scraper.find_safe(By.CSS_SELECTOR, ".inner-content")
    if content:
        print(content.text)
    scraper.back_to_top()
finally:
    scraper.close()