Python最火爬虫框架Scrapy入门与实践，豆瓣电影 Top 250 数据采集

scrapy startproject douban

from scrapy.cmdline import execute
execute(['scrapy','crawl','douban'])

import scrapy
 
class DoubanItem(scrapy.Item):
    num=scrapy.Field() #序列号
    name=scrapy.Field() #电影名
    introduce=scrapy.Field() #介绍
    star=scrapy.Field() # 星级评分
    appraise=scrapy.Field() # 评价人数
    survey=scrapy.Field() #一句话介绍

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)'

# Enable and configure HTTP caching (disabled by default)
# See [url=https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings]https://docs.scrapy.org/en/lates ... middleware-settings[/url]
HTTPCACHE_ENABLED = True
HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

ITEM_PIPELINES = {
    'douban.pipelines.DoubanPipeline': 300,
}

# -*- coding: utf-8 -*-
import scrapy
import requests
from douban.items import DoubanItem
from bs4 import BeautifulSoup
from scrapy.http import Request

class DoubSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['movie.douban.com']
    start_urls=["https://movie.douban.com/top250"]


    def parse(self, response):
        item_list=response.xpath('//ol[@class="grid_view"]/li')
        for item in item_list:
            douban_item=DoubanItem()
            douban_item['num'] = item.xpath('.//div[@class="pic"]/em/text()').extract_first()
            douban_item['name']=item.xpath('.//div[@class="hd"]/a/span[1]/text()').extract_first()
            #print(douban_item['name'])
            introduces=item.xpath('.//div[@class="bd"]/p[1]/text()').extract()
            for introduce in introduces:
                introduce_date="".join(introduce.split())
                douban_item['introduce']=introduce_date
                #print(douban_item['introduce'])
            douban_item['star']=item.xpath('.//div[@class="star"]/span[@class="rating_num"]/text()').extract_first()
            #print(douban_item['star'])
            douban_item['appraise'] = item.xpath('.//div[@class="star"]/span[4]/text()').extract_first()
            douban_item['survey']=item.xpath('.//p[@class="quote"]/span[@class="inq"]/text()').extract_first()
            #print(douban_item['survey'])
            print(douban_item)
            yield douban_item
        next_page=response.xpath('//div[@class="paginator"]/span[@class="next"]/a/@href').extract()
        if next_page:
            yield Request(f'https://movie.douban.com/top250{next_page[0]}',callback=self.parse)

def parse(self, response):
    paginator_urls=[]
    paginator_urls.extend(self.start_urls)
    paginators=BeautifulSoup(response.text,'lxml').find('div',class_="paginator").find_all('a')[:-1]
    for paginator in paginators:
        paginator=f"https://movie.douban.com/top250{paginator['href']}"
        paginator_urls.append(paginator)
    print(paginator_urls)
    paginator_urls=set(paginator_urls)
    for paginator_url in paginator_urls:
        print(paginator_url)
        yield Request(paginator_url,callback=self.get_content)

def get_content(self,response):
    thispage=BeautifulSoup(response.text,'lxml').find('span',class_="thispage").get_text()
    print(thispage)

for v in dllist:
     print(v.xpath("./dt/text()").extract_first())
     print("="*50)
     alist = v.xpath(".//a")
     for a in alist:
             print(a.xpath("./@href").extract_first(),end=":")
             print(a.xpath("./span/img/@alt").extract_first())
             
#来源：CSDN博主「周雄伟」

import pymysql
 
class DoubanPipeline(object):
    def __init__(self):
        #连接MySQL数据库
        self.connect=pymysql.connect(
            host="localhost",
            user="root",
            password="123456",
            db="xiaoshuo",
            port=3306,
        )
        self.cursor=self.connect.cursor()
    def process_item(self, item, spider):
        self.cursor.execute('insert into movie(num,name,introduce,star,appraise,survey)VALUES("{}","{}","{}","{}","{}","{}")'.format(item['num'],item['name'],item['introduce'],item['star'],item['appraise'],item['survey']))
        self.connect.commit()
        return item
 
    #关闭数据库
    def close_spider(self,spider):
        self.cursor.close()

scrapy crawl douban -o douban.csv

scrapy crawl douban -o douban.csv -s FEED_EXPORT_ENCIDING=utf-8

FEED_EXPORT_ENCODING = 'gb18030'