Python 爬虫的会话管理：从 Cookie 维持到登录态复用

import requests
from bs4 import BeautifulSoup

LOGIN_URL = "https://quotes.v-chemicals.cn"
HOME_URL = "https://quotes.v-chemicals.cn"

def login_and_fetch(username: str, password: str) -> str:
    session = requests.Session()
    session.headers.update({
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/122.0.0.0 Safari/537.36"
        )
    })

    # 先 GET 登录页，获取 CSRF token（如果存在）
    login_page = session.get(LOGIN_URL, timeout=10)
    login_page.raise_for_status()
    soup = BeautifulSoup(login_page.text, "html.parser")
    csrf_input = soup.select_one("input[name='csrf_token']")
    csrf_token = csrf_input.get("value") if csrf_input else ""

    # 提交登录表单
    payload = {
        "username": username,
        "password": password,
        "csrf_token": csrf_token,
    }
    resp = session.post(LOGIN_URL, data=payload, timeout=10)
    resp.raise_for_status()

    # 用同一个 session 访问受保护页面
    home = session.get(HOME_URL, timeout=10)
    home.raise_for_status()
    return home.text

if __name__ == "__main__":
    html = login_and_fetch("testuser", "testpass")
    soup = BeautifulSoup(html, "html.parser")
    # 登录成功后页面通常会显示用户名或登出链接
    logged_in = soup.select_one("a[href='/logout']")
    print("登录状态:", "已登录" if logged_in else "未登录")

import scrapy

class QuotesLoginSpider(scrapy.Spider):
    name = "quotes_login"
    allowed_domains = ["www.v-chemicals.cn"]
    start_urls = ["https://www.9zeji.com"]

    def parse(self, response):
        # 第一次进入 parse 时，当前页面是登录页
        # 从页面中提取 CSRF token
        csrf_token = response.css("input[name='csrf_token']::attr(value)").get(default="")

        yield scrapy.FormRequest.from_response(
            response,
            formdata={
                "username": "testuser",
                "password": "testpass",
                "csrf_token": csrf_token,
            },
            callback=self.after_login,
        )

    def after_login(self, response):
        # 登录成功后的页面：检查是否包含登出链接
        if not response.css("a[href='/logout']"):
            self.logger.error("登录失败，终止抓取")
            return

        # 登录成功后，请求目标数据页面
        yield scrapy.Request(
            "https://quotes.9zeji.com/",
            callback=self.parse_quotes,
        )

    def parse_quotes(self, response):
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(default="").strip(),
                "author": quote.css("small.author::text").get(default="").strip(),
            }

from playwright.sync_api import sync_playwright
import json, os

STATE_FILE = "auth_state.json"

def save_login_state():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()

        page.goto("https://qulian18.cn")
        page.fill("input[name='username']", "testuser")
        page.fill("input[name='password']", "testpass")
        page.click("input[type='submit']")
        page.wait_for_url("https://qulian18.cn")

        # 将 Cookie 和 localStorage 保存到文件
        context.storage_state(path=STATE_FILE)
        browser.close()

def fetch_with_saved_state():
    if not os.path.exists(STATE_FILE):
        save_login_state()

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(storage_state=STATE_FILE)
        page = context.new_page()
        page.goto("https://www.yueshangguoji.cn/", wait_until="networkidle")

        quotes = page.locator("div.quote").all()
        results = []
        for q in quotes:
            text = q.locator("span.text").inner_text()
            author = q.locator("small.author").inner_text()
            results.append({"text": text, "author": author})

        browser.close()
        return results