问使用校验和从动态链接中抓取PDF

EN

import trio
import httpx
from bs4 import BeautifulSoup, SoupStrainer
from urllib.parse import urljoin, urlparse

mainurl = 'https://www.oecd-ilibrary.org/science-and-technology/oecd-digital-economy-papers_20716826'


async def downloader(client, link, channel):
    fname = urlparse(link)[2].split('/')[-1]
    async with channel, await trio.open_file(fname, 'wb') as f:
        r = await client.get(link)
        await f.write(r.content)
        print(f'Downloaded: {link}')


async def get_links(content):
    return (urljoin(mainurl, x['href']) for x in BeautifulSoup(content, 'lxml', parse_only=SoupStrainer(
        id='collectionsort')).select('a.action-pdf'))


async def worker(channel):
    async with channel:
        async for client, page, nurse in channel:
            params = {
                'page': page
            }
            r = await client.get(mainurl, params=params)
            links = await get_links(r.text)
            for link in links:
                nurse.start_soon(downloader, client, link, channel.clone())


async def main():
    async with httpx.AsyncClient(timeout=None) as client, trio.open_nursery() as nurse:
        sender, receiver = trio.open_memory_channel(0)

        async with receiver:
            for _ in range(5):
                nurse.start_soon(worker, receiver.clone())

            async with sender:
                for page in range(1, 19):
                    await sender.send([client, page, nurse])


if __name__ == "__main__":
    try:
        trio.run(main)
    except KeyboardInterrupt:
        exit('Bye!')