Scrapy的Meta、异常处理

yield scrapy.FormRequest(meta={"keyword": keyword, "sta_date": sta_date})

#等同于response.request.meta
response.meta

https://docs.scrapy.org/en/latest/topics/request-response.html?highlight=meta#topics-request-meta

request.headers["cookie"] = "......."

request.meta['proxy'] = '127.0.0.1:8989'

# 数字越小, 优先级越高
request.meta['priority'] = 10

    def start_request(self):
        yield scrapy.FormRequest(errorback=self.process_error)
    
    def process_error(self, failure):
        print(failure)
        # 记录异常
        # 发送通知
        # 重做任务
        ...

from scrapy.exceptions import CloseSpider

def process_exception(self, request, exception, spider):
    pass

    def process_exception(self, request, exception, spider):
        # 如果异常是cookie池空了, 可以在这里完成cookie池的补充
        # 补充cookie池
        if isinstance(exception, IndexError):
            # 我已经知道异常产生原因, 所以免除当前重试次数的计数
            retry_times = request.meta.get('retry_times', 1)
            request.meta['retry_times'] = retry_times - 1
            return request