在pipelines.py中从爬行器获取变量

# spiders/example_spider.py
import scrapy

class ExampleSpider(scrapy.Spider):
    name = 'example'
    start_urls = ['http://example.com']

    def parse(self, response):
        for href in response.css('a::attr(href)').getall():
            yield {
                'title': response.css('h1::text').get(),
                'link': href
            }

# pipelines.py
class ExamplePipeline:
    def process_item(self, item, spider):
        # 在这里可以对item进行处理
        title = item.get('title')
        link = item.get('link')
        # 例如，打印标题和链接
        print(f'Title: {title}, Link: {link}')
        return item

# settings.py
ITEM_PIPELINES = {
    'myproject.pipelines.ExamplePipeline': 300,
}

# 在ExamplePipeline中添加日志
import logging

class ExamplePipeline:
    def process_item(self, item, spider):
        logging.info(f'Processing item: {item}')
        title = item.get('title')
        link = item.get('link')
        print(f'Title: {title}, Link: {link}')
        return item