Scrapy1.6 爬虫框架3 分页处理

import scrapy


class BooksSpider(scrapy.Spider):
    name = "books"
    start_urls = [
        'http://books.toscrape.com/',
    ]

    def parse(self, response):
        for book in response.css('article.product_pod'):
            # 选择器可以通过命令行工具就行调试
            yield {
                # xpath 语法 @ATTR 为选中为名ATTR的属性节点
                'name': book.xpath('h3/a/@title').get(),
                'price': book.css('p.price_color::text').get(),
            }

class BooksSpider(scrapy.Spider):
    # 爬取命令 scrapy crawl books
    name = "books"

    start_urls = [
        'http://books.toscrape.com/',
    ]

    def parse(self, response):
        for book in response.css('article.product_pod'):
            yield {
                'name': book.xpath('h3/a/@title').get(),
                'price': book.css('p.price_color::text').get(),
            }

        # 检查分页
        # 提取下一页的链接
        next_url = response.css('ul.pager li.next a::attr(href)').extract_first()
        if next_url:
            next_url = response.urljoin(next_url)
            # 构造新的 Request 对象
            yield scrapy.Request(next_url, callback=self.parse)