python crawlspider 例子

rules=( Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+.htm'),callback='parse_item'), Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+.htm'),follow=True), ) 1、##

-- coding: utf-8 --

import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule import re

class CfSpider(CrawlSpider): name = 'cf' allowed_domains = ['bxjg.circ.gov.cn'] start_urls = ['http://bxjg.circ.gov.cn/web/site0/tab5240/Default.htm']

rules = (
    Rule(LinkExtractor(allow=r'/web/site0/tab5240/info\d+\.htm'), callback='parse_item', ),
    Rule(LinkExtractor(allow=r'/web/site0/tab5240/module14430/page\d+\.htm'),follow=True, ),
)

def parse_item(self, response):
    item = {}
    item['title']=response.xpath('//title/text()').extract()
    item['publish_date'] = response.xpath("//meta[@name='docdate']/@content").extract()
    print(item)

2、settings.py

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.54 Safari/536.5' DOWNLOAD_DELAY=1