Scrapy爬虫框架介绍

pip install scrapy

scrapy startproject jd_crawler_scrapy

import scrapy


class JdSearch(scrapy.Spider):
    name = "jd_search"

    def start_requests(self):
        for keyword in ["鼠标", "键盘", "显卡", "耳机"]:
            for page_num in range(1, 11):
                url = f"https://search.jd.com/Search?keyword={keyword}&page={page_num}"

                # 选用FormRequest是因为它既可以发送GET请求, 又可以发送POST请求
                yield scrapy.FormRequest(
                    url=url,
                    method='GET',
                    # formdata=data,             # 如果是post请求, 携带数据使用formdata参数
                    callback=self.parse_search   # 指定回调函数处理response对象
                )


    def parse_search(self, response):
        print(response)

scrapy crawl spider_name