#Python爬虫#Item Pipeline介绍(附爬取网站获取图片到本地代码)

from scrapy.exceptions import DropItem

class PricePipeline(object):

    vat_factor = 1.15

    def process_item(self, item, spider):
        if item['price']:
            if item['price_excludes_vat']:
                item['price'] = item['price'] * self.vat_factor
            return item
        else:
            raise DropItem("Missing price in %s" % item)

import json

class JsonWriterPipeline(object):

    def open_spider(self, spider):
        self.file = open('items.jl', 'w')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + "\n"
        self.file.write(line)
        return item

import pymongo

class MongoPipeline(object):

    collection_name = 'scrapy_items'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert_one(dict(item))
        return item

from scrapy.exceptions import DropItem

class DuplicatesPipeline(object):

    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        if item['id'] in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(item['id'])
            return item

ITEM_PIPELINES = {
    'myproject.pipelines.PricePipeline': 300,
    'myproject.pipelines.JsonWriterPipeline': 800,
}

scrapy crawl tushu -o tushu.json

ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}

ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}

FILES_STORE = '/path/to/valid/dir'

IMAGES_STORE = '/path/to/valid/dir'

ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1}

import scrapy

class MyItem(scrapy.Item):

    # ... other item fields ...
    image_urls = scrapy.Field()
    images = scrapy.Field()

# 图片下载存储路径
ITEM_STORE = 'E:\\'

# 120天后过期
FILES_EXPIRES = 120

# 30天后过期
IMAGES_EXPIRES = 30

IMAGES_THUMBS = {
    'small': (50, 50),
    'big': (270, 270),
}

IMAGES_MIN_HEIGHT = 110
IMAGES_MIN_WIDTH = 110

# 新增内容从这里开始#################################################################
# 启动pipline
ITEM_PIPELINES = {
    # 注意，如果自定义图片名称时，此条内容要注释，不然自定义图片名不生效
    'scrapy.pipelines.images.ImagesPipeline': 1,
        # 自定义图片名称后，可以取消注释此条
    # 'sp.pipelines.SpDownimagePipeline': 200,
}
# 图片保存地址
IMAGES_STORE = 'E:\\'
# 图片过期时间30天
IMAGES_EXPIRES = 30
# 设置缩略图
# IMAGES_THUMBS = {
#     'small': (50, 50),
#     'big': (270, 270),
# }
# 过滤小图片
# IMAGES_MIN_HEIGHT = 110
# IMAGES_MIN_WIDTH = 110
# 允许重定向
MEDIA_ALLOW_REDIRECTS = True
# 控制时间，下载等待时间3秒
DOWNLOAD_DELAY = 3
# 请求user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'
# 新增内容从这里结束#################################################################

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class SpItem(scrapy.Item):
    """
    定义item字段
    """
    # 网页名称
    image = scrapy.Field()
    # 网页内图片链接
    image_urls = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from sp.items import SpItem

class MyBlog(scrapy.Spider):
    name = 'ftx'
    start_urls = ['http://esf.fang.com']

    def parse(self, response):
        """
        爬取初始start_urls列表项目的url(相对路径)，通过response.follow生成
        request，作为参数传入回掉函数parse_item
        """
        # 获取首页所有二手房的链接地址（相对地址）
        page_urls = response.css("p.title a::attr(href)").extract()
        for page_url in page_urls:
            # 相对地址使用response.follow方式拼接url
            request = response.follow(page_url, callback=self.parse_item)
            # 如果获取的连接是绝对地址，用下面这条方法
            # request = scrapy.Request(page_url, callback=self.parse_item)
            yield request

    def parse_item(self, response):
        """
        处理item函数
        :param response: 请求的网页内容
        :return: item
        """
        # 导入item类
        item = SpItem()
        # 每个个人页面中的图片
        # image是每个详细页的标题， image_urls是每个详细页内图片的url
        item['image'] = response.css("div.floatl::text").extract_first().strip()
        item['image_urls'] = response.css("img.loadimg::attr(data-src)").extract()
        yield item

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import scrapy
from scrapy.pipelines.images import ImagesPipeline

class SpDownimagePipeline(ImagesPipeline):
    """
    自定义图片下载类
    """
    def get_media_requests(self, item, info):
        """
        ImagesPipeline类的方法,必须返回每个图像URL的Request
        :param item:获取的item
        """
        # 从item中获取图片url并发送请求，image_urls就是items.py中定义的字段
        for image_url in item['image_urls']:
            # meta作用就是可以将item的值传给下一个函数使用，类似于先缓存起来
            yield scrapy.Request(image_url, meta={'item': item})

    def item_completed(self, results, item, info):
        """
        此处没有做修改，只是把ImagesPipeline的方法拿过来用，必须返回item
        """
        if isinstance(item, dict) or self.images_result_field in item.fields:
            item[self.images_result_field] = [x for ok, x in results if ok]
        return item

    def file_path(self, request, response=None, info=None):
        """
        file_path为ImagePipeline自带的方法，这里我们重写这个方法，
        为了能够自定义图片的名称，如果不重写，SHA1 hash格式，类似full/63bbfea82b8880ed33cdb762aa11fab722a90a24.jpg
        """
        # 获取item，从get_media_requests的Request中获取
        item = request.meta['item']
        # 图片名称，一版用split（‘/’）分割后取最后一个值也就是-1，这里没用-1是因为图片最后一个字段不是随机数
        # 是长乘以宽如：452x340c.jpg，容易重名，所以用的-2，倒数第二个字段
        image_guid = request.url.split('/')[-2] + '.jpg'
        # 全名，包括路径和图片名
        fullname = "full/%s/%s" % (item['image'], image_guid)
        return fullname

def get_media_requests(self, item, info):
    for file_url in item['file_urls']:
        yield scrapy.Request(file_url)

[(True,
  {'checksum': '2b00042f7481c7b056c4b410d28f33cf',
   'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
   'url': 'http://www.example.com/files/product1.pdf'}),
 (False,
  Failure(...))]

from scrapy.exceptions import DropItem

def item_completed(self, results, item, info):
    file_paths = [x['path'] for ok, x in results if ok]
    if not file_paths:
        raise DropItem("Item contains no files")
    item['file_paths'] = file_paths
    return item

import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem

class MyImagesPipeline(ImagesPipeline):

    def get_media_requests(self, item, info):
        for image_url in item['image_urls']:
            yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        item['image_paths'] = image_paths
        return item