Python + BeautifulSoup4 爬虫入门

pip install requests beautifulsoup4 lxml

import csv
import time
import random
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "http://wlt.yunhaizz.cn"
HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/120.0.0.0 Safari/537.36"
    ),
}

def fetch_page(url: str) -> str | None:
    """获取网页源码，失败返回 None"""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=10)
        resp.raise_for_status()
        resp.encoding = resp.apparent_encoding
        return resp.text
    except Exception as exc:
        print(f"请求失败: {url}, 错误: {exc}")
        return None

def parse_books(html: str) -> list[dict]:
    """解析一页书籍信息"""
    soup = BeautifulSoup(html, "lxml")
    books = []

    # 每本书通常位于 article.product_pod 中
    items = soup.select("article.product_pod")

    for item in items:
        title_tag = item.select_one("h3 a")
        price_tag = item.select_one(".price_color")
        link_tag = item.select_one("h3 a")

        title = title_tag.get("title", "").strip() if title_tag else ""
        price = price_tag.get_text(strip=True) if price_tag else ""
        href = link_tag.get("href", "") if link_tag else ""
        url = urljoin(BASE_URL, href) if href else ""

        books.append({
            "title": title,
            "price": price,
            "url": url,
        })

    return books

def save_to_csv(books: list[dict], filename: str = "books.csv") -> None:
    """把结果保存为 CSV"""
    if not books:
        print("没有数据可保存")
        return

    with open(filename, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.DictWriter(f, fieldnames=["title", "price", "url"])
        writer.writeheader()
        writer.writerows(books)

    print(f"已保存 {len(books)} 条数据到 {filename}")

def main():
    url = f"{BASE_URL}/catalogue/page-1.html"
    html = fetch_page(url)

    if not html:
        return

    books = parse_books(html)

    for book in books:
        print(book)

    save_to_csv(books)

    # 避免请求过快
    time.sleep(random.uniform(1, 2))

if __name__ == "__main__":
    main()

{'title': 'A Light in the ...', 'price': '£51.77', 'url': 'http://wlt.yunhaizz.cn/catalogue/a-light-in-the-...'}
{'title': 'Tipping the Velvet', 'price': '£53.74', 'url': 'http://wlt.yunhaizz.cn/catalogue/tipping-the-velvet_999/index.html'}
...
已保存 20 条数据到 books.csv