day134-scrapy的post请求&回调函数参数传递&代理池&并发

class PicproDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.
    # 设置代理配置
    
    proxy_http = [
        'http://209.34.27.9:8181',
        'http://209.34.28.9:8181',
        'http://209.34.29.9:8181'
    ]
    proxy_https = [
        'https://119.57.84.58:8080',
        'https://119.58.84.58:8080',
        'https://119.59.84.58:8080'
    ]
    user_agent_list = [
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "
        "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
        "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "
        "(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "
        "(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
        "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "
        "(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"]

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        """
        在请求发起请求是改变代理
        """
        if request.url.split(':')[0] == 'http':
            request.meta['proxy'] = random.choice(self.proxy_http)
        else:
            request.meta['proxy'] = random.choice(self.proxy_https)

        request.headers['User-Agent'] = random.choice(self.user_agent_list)
        print(request.headers['User-Agent'])

        return None

# 增加并发：
#     默认scrapy开启的并发线程为32个，可以适当进行增加。
#     在settings配置文件中修改CONCURRENT_REQUESTS = 100值为100,并发设置成了为100。
#
# 降低日志级别：
#     在运行scrapy时，会有大量日志信息的输出，为了减少CPU的使用率。
#     可以设置log输出信息为INFO或者ERROR即可。在配置文件中编写：LOG_LEVEL = ‘INFO’
#
# 禁止cookie：
#     如果不是真的需要cookie，则在scrapy爬取数据时可以禁止cookie从而减少CPU的使用率，提升爬取效率。
#     在配置文件中编写：COOKIES_ENABLED = False
#
# 禁止重试：
#     对失败的HTTP进行重新请求（重试）会减慢爬取速度，因此可以禁止重试。
#     在配置文件中编写：RETRY_ENABLED = False
#
# 减少下载超时：
#     如果对一个非常慢的链接进行爬取，减少下载超时可以能让卡住的链接快速被放弃，从而提升效率。
#     在配置文件中进行编写：DOWNLOAD_TIMEOUT = 10 超时时间为10s

CONCURRENT_REQUESTS = 10
LOG_LEVEL = 'ERROR'
COOKIES_ENABLED = False
RETRY_ENABLED = False
DOWNLOAD_TIMEOUT = 5