使用Pyppeteer抓取渲染网页

import asyncio
from pyppeteer import launch

async def main():
    # headless参数设为False，则变成有头模式
    browser = await launch(
        # headless=False
    )
    
    page = await browser.newPage()
    
    # 设置页面视图大小
    await page.setViewport(viewport={'width':1280, 'height':800})
    
    # 是否启用JS，enabled设为False，则无渲染效果
    await page.setJavaScriptEnabled(enabled=True)
    
    await page.goto('https://www.toutiao.com/')
    
    # 打印页面cookies
    print(await page.cookies())
    
    # 打印页面文本
    print(await page.content())
    
    # 打印当前页标题
    print(await page.title())
    
    # 抓取新闻标题
    title_elements = await page.xpath('//div[@class="title-box"]/a')
    for item in title_elements:
        # 获取文本
        title_str = await (await item.getProperty('textContent')).jsonValue()
        print(await item.getProperty('textContent'))
        # 获取链接
        title_link = await (await item.getProperty('href')).jsonValue()
        print(title_str)
        print(title_link)
    
    # 关闭浏览器
    await browser.close()

asyncio.get_event_loop().run_until_complete(main())

# 1 打开一个网页并做截图
# 首次运行示例时，pyppeteer会自动下载对应操作系统的chromium
import asyncio
from pyppeteer import launch

async def main():
    browser = await launch()
    page = await browser.newPage()
    await page.goto('http://example.com')
    await page.screenshot({'path': 'example.png'})
    await browser.close()

asyncio.get_event_loop().run_until_complete(main())

# 2 在网页上执行一段脚本
import asyncio
from pyppeteer import launch

async def main():
    browser = await launch()
    page = await browser.newPage()
    await page.goto('http://example.com')
    await page.screenshot({'path': 'example.png'})

    dimensions = await page.evaluate('''() => {
        return {
            width: document.documentElement.clientWidth,
            height: document.documentElement.clientHeight,
            deviceScaleFactor: window.devicePixelRatio,
        }
    }''')

    print(dimensions)
    # >>> {'width': 800, 'height': 600, 'deviceScaleFactor': 1}
    await browser.close()

asyncio.get_event_loop().run_until_complete(main())

# Puppeteer只支持字典传参
browser = await launch({'headless': True})

# Pyppeteer支持字典和关键字传参
browser = await launch({'headless': True})
browser = await launch(headless=True)

# Puppeteer使用$符
Page.$()/Page.$$()/Page.$x()

# Pyppeteer使用Python风格的函数名
Page.querySelector()/Page.querySelectorAll()/Page.xpath()
# 简写方式为：
Page.J(), Page.JJ(), and Page.Jx()

content = await page.evaluate('document.body.textContent', force_expr=True)

element = await page.querySelector('h1')
title = await page.evaluate('(element) => element.textContent', element)