python爬虫之微打赏（scrapy版）创建项目itemssettingsweidashangspider保存为csv文件

scrapy startproject weidashang
cd weidashang
scrapy genspider weidashangspider wds.modian.com

import scrapy

class WeidashangItem(scrapy.Item):
    name = scrapy.Field()
    nickname = scrapy.Field()
    money = scrapy.Field()
    pass

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'

import scrapy
import json
import math
from weidashang.items import WeidashangItem


class WeidashangspiderSpider(scrapy.Spider):
    name = "weidashangspider"
    allowed_domains = ["wds.modian.com"]
    start_urls = ['http://wds.modian.com/']

    def start_requests(self):
        for i in range(1, 10):
            params = {
                'ajaxtype': '1',
                'page': str(i),
                'category': '1',
                'pageSize': '8'
            }
            yield scrapy.FormRequest(url='https://wds.modian.com/ajax_first',formdata=params,callback=self.parse)

    def parse(self, response):
        json_data = json.loads(response.text)
        des = json_data['des']
        for data in des:
            name = data['name']
            id = data['id']
            pay_count = data['pay_count']
            all_page = math.ceil(int(pay_count) / 20)
            for i in range(1, int(all_page) + 1):
                params = {
                    'pro_id': id,
                    'type': '1',
                    'page': str(i),
                    'pageSize': '20'
                }
                yield scrapy.FormRequest(url='https://wds.modian.com/ajax_backer_list',formdata=params,meta={'name':name},callback=self.parse_item)

    def parse_item(self,response):
        item = WeidashangItem()
        item['name'] = response.meta['name']
        json_data = json.loads(response.text)
        datas = json_data['data']
        for data in datas:
            nickname = data['nickname']
            money = data['total_back_amount']
            item['nickname'] = nickname
            item['money'] = money
            yield item

cd weidashang
scrapy crawl weidashangspider -o weidashang.csv