Scrapy 爬虫 --- 创建

pip install scrapy

scrapy startproject newspider

第一层[一级 newspider目录]:
  -scrapy.cfg: 配置文件，不需要修改
  -newspider: 文件夹：第二层解释

#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
# @Time : 2019/4/29 16:11 
# @Author : zhao.jia
# @Site :  
# @File : zhilianspider.py 
# @Software: PyCharm

import scrapy
import json
from spider_work.items import ZhaopingItem
import re


class ZhaopinSpider(scrapy.Spider):
    # 爬虫的名字，以此来启动爬虫
    name = 'job_spider'
    # 起始URL，
    baseUrl = 'https://fe-api.zhaopin.com/c/i/sou?start={0}&pageSize=90&cityId=489&kw={1}&kt=3'

    offset = 0  # 偏移量

    def start_requests(self):
        # 打开关键词的json文件
        with open('keywords.json', 'r', encoding='utf8') as f:
            keywords_list = json.load(f)

        start_urls = []

        for item in keywords_list:
            for key, value in item.items():
                for job_key in value:
                    print(job_key)
                    start_urls.append(self.baseUrl.format(str(self.offset), job_key))

        if start_urls is not None:
            for url in start_urls:
                print("start_url:", url)
                yield scrapy.Request(url=url, callback=self.parse, meta={'start_url': url})

    def parse(self, response):
        '''
        页面解析函数
        :param response:
        :return:
        '''
        try:
            data_list = json.loads(response.body)['data']['results']

            if len(data_list) == 0:
                return
            for data in data_list:
                item = ZhaopingItem()
                item["jobType"] = data['jobType']['display']  # 职位所属种类
                item["jobName"] = data['jobName']  # 职位名称
                item["emplType"] = data['emplType']  # 工作类型(兼职、全职)
                item["eduLevel"] = data['eduLevel']['name']  # 学历要求
                item["companyName"] = data['company']['name']  # 公司名称
                item["salary"] = data['salary']  # 薪资
                item["welfare"] = ','.join(data['welfare'])  # 员工福利
                item["city"] = data['city']['display']  # 工作城市
                item["workingExp"] = data['workingExp']['name']  # 工作经验
                # item['infoComLink']=data['company']['url']     #公司详情连接
                # item['positionUrl']=data['positionURL']        # 职位详情链接

                yield item
        except Exception as e:
            print(e)
        try:
            init_url = response.meta['start_url']
            self.offset += 90
            str_offset = str(self.offset)
            pattern = 'start=(.*?)&'
            replace_str = 'start=' + str_offset + '&'
            url = re.sub(pattern=pattern, repl=replace_str, string=init_url)
            yield scrapy.Request(url=url, callback=self.parse)
        except Exception as e:
            print(e)

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class ZhaopingItem(scrapy.Item):

    jobType = scrapy.Field()  # 工作种类（大类:互联网、金融……）
    jobName = scrapy.Field()  # 职位名称
    emplType = scrapy.Field()  # 工作类型(全职、兼职)
    eduLevel = scrapy.Field()  # 学历要求
    companyName = scrapy.Field()  # 公司名称
    salary = scrapy.Field()  # 工资
    welfare = scrapy.Field()  # 福利
    city = scrapy.Field()  # 城市
    workingExp = scrapy.Field()  # 要求
    # infoComLink = scrapy.Field()  # 公司详情连接
    # positionUrl=scrapy.Field() # 职位详情链接

class SpiderWorkPipeline(object):

    """
    存储数据
    """
    pass
    # def __init__(self):
    #     # 连接数据库
    #     self.connect = pymysql.connect(
    #         host=MYSQL_HOST,
    #         db=MYSQL_DBNAME,
    #         user=MYSQL_USER,
    #         passwd=MYSQL_PASSWD,
    #         # charset='utf8',
    #         use_unicode=True)
    #
    #     # 通过cursor执行增删查改
    #     self.cursor = self.connect.cursor()

ROBOTSTXT_OBEY = False

MYSQL_HOST = 'ip'
MYSQL_DBNAME = 'dbname'
MYSQL_USER = 'root'
MYSQL_PASSWD = 'pass'

scrapy crawl job_spider