Python爬虫入门教程 33-100 《海王》评论数据抓取 scrapy

http://m.maoyan.com/mmdb/comments/movie/249342.json?_v_=yes&offset=15&startTime=2018-12-11%2009%3A58%3A43

url:http://m.maoyan.com/mmdb/comments/movie/249342.json
offset:15
startTime:起始时间

import scrapy
import json
from haiwang.items import HaiwangItem

class HaiwangSpider(scrapy.Spider):
    name = 'Haiwang'
    allowed_domains = ['m.maoyan.com']
    start_urls = ['http://m.maoyan.com/mmdb/comments/movie/249342.json?_v_=yes&offset=0&startTime=0']

    def parse(self, response):
        print(response.url)
        body_data = response.body_as_unicode()

        js_data = json.loads(body_data)
        item = HaiwangItem()
        for info in js_data["cmts"]:

            item["nickName"] = info["nickName"]
            item["cityName"] = info["cityName"] if "cityName" in info else ""
            item["content"] = info["content"]
            item["score"] = info["score"]
            item["startTime"] = info["startTime"]
            item["approve"] = info["approve"]
            item["reply"] = info["reply"]
            item["avatarurl"] = info["avatarurl"]

            yield item

        yield scrapy.Request("http://m.maoyan.com/mmdb/comments/movie/249342.json?_v_=yes&offset=0&startTime={}".format(item["startTime"]),callback=self.parse)

DEFAULT_REQUEST_HEADERS = {
    "Referer":"http://m.maoyan.com/movie/249342/comments?_v_=yes",
    "User-Agent":"Mozilla/5.0 Chrome/63.0.3239.26 Mobile Safari/537.36",
    "X-Requested-With":"superagent"
}

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 1
# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'haiwang.pipelines.HaiwangPipeline': 300,
}

import scrapy


class HaiwangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    nickName = scrapy.Field()
    cityName = scrapy.Field()
    content = scrapy.Field()
    score = scrapy.Field()
    startTime = scrapy.Field()
    approve = scrapy.Field()
    reply =scrapy.Field()
    avatarurl = scrapy.Field()

import os
import csv


class HaiwangPipeline(object):
    def __init__(self):
        store_file = os.path.dirname(__file__) + '/spiders/haiwang.csv'
        self.file = open(store_file, "a+", newline="", encoding="utf-8")
        self.writer = csv.writer(self.file)

    def process_item(self, item, spider):
        try:
            self.writer.writerow((
                item["nickName"],
                item["cityName"],
                item["content"],
                item["approve"],
                item["reply"],
                item["startTime"],
                item["avatarurl"],
                item["score"]
            ))

        except Exception as e:
            print(e.args)

        def close_spider(self, spider):
            self.file.close()

from scrapy import cmdline
cmdline.execute(("scrapy crawl Haiwang").split())