【python爬虫学习 】python

scrapy startproject tutorial

scrapy genspider baidu www.baidu.com

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class BaiduItems(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    link = scrapy.Field()
    desc = scrapy.Field()
    pass

# -*- coding: utf-8 -*-
import scrapy
# 引入数据容器
from tutorial.BaiduItems import BaiduItems

class BaiduSpider(scrapy.Spider):
    name = 'baidu'
    allowed_domains = ['www.readingbar.net']
    start_urls = ['http://www.readingbar.net/']
    def parse(self, response):
        for sel in response.xpath('//ul/li'):
            item = BaiduItems()
            item['title'] = sel.xpath('a/text()').extract()
            item['link'] = sel.xpath('a/@href').extract()
            item['desc'] = sel.xpath('text()').extract()
            yield item
        pass

# 设置用户代理
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'

# 解决 robots.txt 相关debug
ROBOTSTXT_OBEY = False
# scrapy 解决数据保存乱码问题
FEED_EXPORT_ENCODING = 'utf-8'

 scrapy crawl baidu -o baidu.json
 

# -*- coding: utf-8 -*-
import scrapy

from scrapyProject.BaiduItems import BaiduItems

class BaiduSpider(scrapy.Spider):
    name = 'baidu'
    # 由于tab包含其他域名,需要添加域名否则无法爬取
    allowed_domains = [
        'www.baidu.com',
        'v.baidu.com',
        'map.baidu.com',
        'news.baidu.com',
        'tieba.baidu.com',
        'xueshu.baidu.com'
    ]
    start_urls = ['https://www.baidu.com/']
    def parse(self, response):
        item = BaiduItems()
        item['title'] = response.xpath('//title/text()').extract()
        yield item
        for sel in response.xpath('//a[@class="mnav"]'):
            item = BaiduItems()
            item['nav'] = sel.xpath('text()').extract()
            item['href'] = sel.xpath('@href').extract()
            yield item
            # 根据提取的nav地址建立新的请求并执行回调函数
            yield scrapy.Request(item['href'][0],callback=self.parse_newpage)
        pass
    # 深度提取tab网页标题信息
    def parse_newpage(self, response):
        item = BaiduItems()
        item['title'] = response.xpath('//title/text()').extract()
        yield item
        pass