Amazon图片下载器：利用Scrapy库完成图像下载任务

scrapy startproject amazon_image_downloader

amazon_image_downloader/
    scrapy.cfg            # 配置文件
    amazon_image_downloader/     # 项目的Python模块
        __init__.py

        items.py          # 项目中的item文件

        middlewares.py    # 项目中的中间件文件

        pipelines.py      # 项目中的管道文件

        settings.py       # 项目的设置文件

        spiders/          # 存放爬虫代码的目录
            __init__.py

import scrapy


class AmazonImageItem(scrapy.Item):
    # 定义一个Item类，用来存储图片的URL和名称
    image_urls = scrapy.Field() # 图片的URL列表
    image_name = scrapy.Field() # 图片的名称

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from amazon_image_downloader.items import AmazonImageItem


class AmazonSpider(CrawlSpider):
    # 定义一个CrawlSpider类，用来实现自动跟进链接的功能
    name = 'amazon_spider' # 爬虫的名称
    allowed_domains = ['amazon.com'] # 允许爬取的域名列表
    start_urls = ['https://www.amazon.com/s?k=book'] # 起始URL列表

    rules = (
        # 定义规则列表，指定如何从响应中提取链接并跟进
        Rule(LinkExtractor(allow=r'/s\?k=book&page=\d+'), follow=True), # 匹配商品列表页的链接，并跟进
        Rule(LinkExtractor(allow=r'/dp/\w+'), callback='parse_item'), # 匹配商品详情页的链接，并调用parse_item函数
    )

    def parse_item(self, response):
        # 定义解析函数，从响应中提取数据并生成Item对象
        item = AmazonImageItem() # 创建一个Item对象
        item['image_urls'] = [response.xpath('//img[@id="imgBlkFront"]/@src')
                              .get()] # 从响应中提取图片的URL，并存入image_urls字段
        item['image_name'] = response.xpath('//span[@id="productTitle"]/text()')
                              .get().strip() # 从响应中提取图片的名称，并存入image_name字段
        return item # 返回Item对象

# Scrapy settings for amazon_image_downloader project

assistant = 'amazon_image_downloader'

SPIDER_MODULES = ['amazon_image_downloader.spiders']
NEWSPIDER_MODULE = 'amazon_image_downloader.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'amazon_image_downloader (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'scrapy.pipelines.images.ImagesPipeline': 300, # 启用图片管道，并指定优先级为300
}

# Configure images pipeline
# See https://docs.scrapy.org/en/latest/topics/images.html
IMAGES_STORE = 'images' # 指定图片管道使用的本地存储路径为images文件夹
IMAGES_URLS_FIELD = 'image_urls' # 指定图片管道使用的Item字段为image_urls
IMAGES_RESULT_FIELD = 'image_results' # 指定图片管道使用的Item字段为image_results

# Configure downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 100, # 启用代理中间件，并指定优先级为100
}

# Configure proxy pool
# 亿牛云代理 https://www.16yun.cn
PROXY_POOL = [
    'http://username:password@domain:port', # 使用亿牛云爬虫代理提供的域名、端口、用户名、密码
    'http://username:password@domain:port',
    ...
]

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item exporters
#

# Configure concurrent requests and download delay
# See https://docs.scrapy.org/en/latest/topics/settings.html
CONCURRENT_REQUESTS = 16 # 设置Scrapy downloader 并发请求的最大值为16
CONCURRENT_REQUESTS_PER_DOMAIN = 8 # 设置对单个网站进行并发请求的最大值为8
DOWNLOAD_DELAY = 0.5 # 设置下载两个页面之间等待的时间为0.5秒