爬虫爬取智联招聘的职位信息（基础）

andrew_a

发布于 2019-07-30 13:07:44

6880

发布于 2019-07-30 13:07:44

文章被收录于专栏：Python爬虫与数据分析Python爬虫与数据分析

简单爬取智联招聘的内容，爬之前在网上找关于这方面的代码，最后发现都不行，智联现在的招聘信息都是js加载，直接请求响应不到响应的内容，只能使用selenium，通过selenium加载，网页加载完成后返回page_source。

selenium放在中间件中。

开始正文

先找到要爬取的url,https://sou.zhaopin.com/?pageSize=60&jl=854&kw=java%E5%B7%A5%E7%A8%8B%E5%B8%88&kt=3都是在搜索页面根据条件查找自己想要的信息，我的条件就是西安的java开发。取前10页的数据，每页60条

第一步就是创建项目；

 scrapy startproject zhilian

打开该项目：（项目目录）

编写selenium中间件

class SeleniumSpiderMiddleware(object):

    def process_request(self, request, spider):

        try:
            spider.browser.get(request.url)
        except TimeoutException as e:
            print('超时')
            spider.browser.execute_script('window.stop()')
        time.sleep(2)
        return HtmlResponse(url=spider.browser.current_url, body=spider.browser.page_source,
                            encoding="utf-8", request=request)

这里的中间件只是做一个请求，返回加载js之后的页面。如果用默认的中间件会拿不到你需要的数据。

在spider目录下创建

zhilain_spiders.py

初始化，打开浏览器，设置超时时间，请求会去中间件执行get方法

def __init__(self):

        self.browser = webdriver.Chrome()
        # self.browser.fullscreen_window()
        self.browser.set_page_load_timeout(30)
        self.urls = []
        for i in range(0, 10):
            self.url = 'https://sou.zhaopin.com/?p={}&pageSize=60&jl=854&kw=java%E5%B7%A5%E7%A8%8B%E5%B8%88&kt=3'.format(str(i))
            self.urls.append(self.url)


    def start_requests(self):

        for i in range(len(self.urls)):

            yield Request(url=self.urls[i], callback=self.parse)

贴代码

zhilain_spiders.py

----------------------------------------------------------------

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/9/30 11:08
# @Author  : jia.zhao
# @Desc    : 
# @File    : zhilain_spiders.py
# @Software: PyCharm


from zhilian.items import ZhaopinItem
from scrapy import Spider, Request
from selenium import webdriver
import time

class ZhaopinSpider(Spider):
    name = 'zhilian'

    allowed_domains = ['www.zhaopin.com']
    start_urls = ['http://www.zhaopin.com/']
    #start_urls = ['http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E4%B8%8A%E6%B5%B7&kw=java%E5%B7%A5%E7%A8%8B%E5%B8%88&sm=0&sg=720f662a0e894031b9b072246ac2f919&p=1']

    def __init__(self):

        self.browser = webdriver.Chrome()
        # self.browser.fullscreen_window()
        self.browser.set_page_load_timeout(30)
        self.urls = []
        for i in range(0, 10):
            self.url = 'https://sou.zhaopin.com/?p={}&pageSize=60&jl=854&kw=java%E5%B7%A5%E7%A8%8B%E5%B8%88&kt=3'.format(str(i))
            self.urls.append(self.url)


    def start_requests(self):

        for i in range(len(self.urls)):

            yield Request(url=self.urls[i], callback=self.parse)

    def parse(self, response):

        item = ZhaopinItem()

        # 职位
        job_name = response.selector.xpath('//div[@class="listItemBox clearfix"]/div[@class="infoBox"]/div[@class="itemBox nameBox"]/div[@class="jobName"]/a/span[@class="job_title"]')
        job_name = job_name.xpath('string(.)').extract()
        # url
        job_url = response.selector.xpath('//div[@class="listItemBox clearfix"]/div[@class="infoBox"]/div[@class="itemBox nameBox"]/div[@class="jobName"]/a/@href').extract()
        # 公司名称
        company_name = response.selector.xpath('//div[@class="listItemBox clearfix"]/div[@class="infoBox"]/div[@class="itemBox nameBox"]/div[@class="commpanyName"]/a/text()').extract()
        # 薪资范围
        job_salary = response.selector.xpath('//div[@class="listItemBox clearfix"]/div[@class="infoBox"]/div[@class="itemBox descBox"]/div[@class="jobDesc"]/p[@class="job_saray"]/text()').extract()
        # 获取公司的简要地址以及基本要求
        # job_demand = response.selector.xpath('//div[@class="listItemBox clearfix"]/div[@class="infoBox"]/div[@class="itemBox descBox"]/div[@class="jobDesc"]/ul[@class="job_demand"]')
        # for i in job_demand:
        #     jsb = i.xpath('string(.)').extract()
        #     for j in jsb:
        #         print(j)
        #         # companj.split(' ')[0]
        web = webdriver.Chrome()
        # 职位简介列表
        job_summary = []
        for i in range(0, len(job_name)):

            web.get(job_url[i])
            time.sleep(5)
            # 公司福利
            welfare = web.find_elements_by_xpath('//div[@class="welfare-tab-box"]/span')
            # 职位简介
            job_info_list = web.find_elements_by_xpath('//div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/ul[@class="terminal-ul clearfix"]/li')
            for job_info in job_info_list:
                job_summary.append(job_info.text)
            # 职位描述
            current = web.find_elements_by_xpath(
                '//div[@class="terminalpage clearfix"]/div[@class="terminalpage-left"]/div[@class="terminalpage-main clearfix"]/div[@class="tab-cont-box"]/div[@class="tab-inner-cont"]')
            # 公司简介
            company_profile = web.find_elements_by_xpath(
                '//div[@class="terminalpage clearfix"]/div[@class="terminalpage-right"]/div[@class="company-box"]')

            item['jobname'] = job_name[i]
            item['salary'] = job_salary[i]
            item['company_name'] = company_name[i]
            welfare_list = []
            job_list = []
            job_current_list = []
            company_profile_list = []

            for wel in welfare:
                welfare_list.append(wel.text)
            for job_info in job_info_list:
                job_list.append(job_info.text)
            for c in current:
                job_current_list.append(c.text)
            for profile in company_profile:
                company_profile_list.append(profile.text)

            job = '【职位】' + job_name[i] + '\n' + '【薪资范围】' + job_salary[i] + '\n【公司名称】' + company_name[i] + '\n【福利待遇】' + ' '.join(welfare_list) + '\n********\n' + '\n'.join(job_list) + '\n********\n【职位描述】\n' + ','.join(job_current_list) + '\n【公司介绍】\n' + ''.join(company_profile_list) + '\n' + '***********************************************************\n'
            with open('job.txt', 'a', encoding='utf8') as f:
                f.write(job)
            item['welfare'] = ','.join(welfare_list)
            item['job_require'] = ','.join(job_list)
            item['job_desc'] = ''.join(job_current_list)
            item['company_profile'] = ''.join(company_profile_list)
            yield item

        web.close()
        web.quit()

    def closed(self, spider):
        print("spider closed")
        self.browser.close()
        self.browser.quit()

setting.py

-----------------------------------------------------------------------------

最后将请求的数据通过管道，保存到MongoDB数据库

pipelines.py

from zhilian import settings
import pymongo

class ZhilianPipeline(object):


    def __init__(self):

        self.connection = pymongo.MongoClient(settings.MONHOST, settings.MONPORT, connect=False)
        # 如果mongodb数据库有权限，设置用户名和密码
        # self.connection.admin.authenticate(settings.MONGO_USER, settings.MONGO_PSW)
        db = self.connection[settings.MONDB]
        self.collection1 = db[settings.COLLECTION1]


    def process_item(self, item, spider):

        item = dict(item)
        self.collection1.insert_one(item)

        return item