Scrapy框架: 通用爬虫之XMLFeedSpider

scrapy startproject xmlfeedspider

scrapy genspider -t xmlfeed jobbole jobbole.com

import scrapy

class JobboleItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 文章标题
    title = scrapy.Field()
    # 发表日期
    public_date = scrapy.Field()
    # 文章链接
    link = scrapy.Field()

# -*- coding: utf-8 -*-
from scrapy.spiders import XMLFeedSpider
# 导入item
from xmlfeedspider.items import JobboleItem

class JobboleSpider(XMLFeedSpider):
    name = 'jobbole'
    allowed_domains = ['jobbole.com']
    start_urls = ['http://top.jobbole.com/feed/']
    iterator = 'iternodes'  # 迭代器，不指定的话默认是iternodes
    itertag = 'item'  # 抓取item节点

    def parse_node(self, response, selector):
        item = JobboleItem()
        item['title'] = selector.css('title::text').extract_first()
        item['public_date'] = selector.css('pubDate::text').extract_first()
        item['link'] = selector.css('link::text').extract_first()
        return item