21天打造分布式爬虫-Selenium爬取拉钩职位信息(六)

6.1.爬取第一页的职位信息

第一页职位信息

from selenium import webdriver
from lxml import etree
import re
import time

class LagouSpider(object):
    def __init__(self):
        self.driver = webdriver.Chrome()
        #python职位
        self.url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput='
        self.position = []

    def run(self):
        self.driver.get(self.url)
        source = self.driver.page_source
        self.parse_list_page(source)

    def parse_list_page(self,source):
        html = etree.HTML(source)
        links = html.xpath("//a[@class='position_link']/@href")
        #每一页的所有职位的详情url
        for link in links:
            self.request_detail_page(link)
            time.sleep(1)

    def request_detail_page(self,url):
        self.driver.get(url)
        #获取职位详情页的源代码
        source = self.driver.page_source
        self.parse_detail_page(source)

    def parse_detail_page(self,source):
        html = etree.HTML(source)
        position_name = html.xpath("//span[@class='name']/text()")[0]
        job_request_spans = html.xpath("//dd[@class='job_request']//span")
        salary = job_request_spans[0].xpath('.//text()')[0].strip()
        city = job_request_spans[1].xpath('.//text()')[0].strip()
        city = re.sub(r"[\s/]","",city)
        work_years = job_request_spans[2].xpath('.//text()')[0].strip()
        work_years = re.sub(r"[\s/]","",work_years)
        education = job_request_spans[3].xpath('.//text()')[0].strip()
        education = re.sub(r"[\s/]","",education)
        desc = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip()
        position = {
            'name':position_name,
            'salary':salary,
            'city': city,
            'work_years': work_years,
            'education': education,
            'desc': desc,
        }
        self.position.append(position)
        print(position)
        print('-'*200)

if __name__ == '__main__':
    spider = LagouSpider()
    spider.run()

6.2.爬取所有页的职位信息

from selenium import webdriver
from lxml import etree
import re
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

class LagouSpider(object):
    def __init__(self):
        self.driver = webdriver.Chrome()
        #python职位
        self.url = 'https://www.lagou.com/jobs/list_python?city=%E5%8C%97%E4%BA%AC&cl=false&fromSearch=true&labelWords=&suginput='
        self.position = []

    def run(self):
        self.driver.get(self.url)
        while True:
            source = self.driver.page_source
            WebDriverWait(driver=self.driver,timeout=20).until(
                EC.presence_of_element_located((By.XPATH,"//div[@class='pager_container']/span[last()]"))
            )
            self.parse_list_page(source)
            #点“下一页”
            next_btn = self.driver.find_element_by_xpath(
                "//div[@class='pager_container']/span[last()]")
            if "pager_next_disabled" in next_btn.get_attribute("class"):
                break
            else:
                next_btn.click()
            time.sleep(1)


    def parse_list_page(self,source):
        html = etree.HTML(source)
        links = html.xpath("//a[@class='position_link']/@href")
        #每一页的所有职位的详情url
        for link in links:
            self.request_detail_page(link)
            time.sleep(1)

    def request_detail_page(self,url):
        # self.driver.get(url)
        self.driver.execute_script("window.open('%s')"%url)
        self.driver.switch_to.window(self.driver.window_handles[1])

        WebDriverWait(driver=self.driver,timeout=20).until(
            EC.presence_of_element_located((By.XPATH,"//div[@class='job-name']/span[@class='name']"))
        )
        #获取职位详情页的源代码
        source = self.driver.page_source
        self.parse_detail_page(source)
        #关闭当前详情页,并且切换到列表页
        self.driver.close()
        self.driver.switch_to.window(self.driver.window_handles[0])

    def parse_detail_page(self,source):
        html = etree.HTML(source)
        position_name = html.xpath("//span[@class='name']/text()")[0]
        job_request_spans = html.xpath("//dd[@class='job_request']//span")
        salary = job_request_spans[0].xpath('.//text()')[0].strip()
        city = job_request_spans[1].xpath('.//text()')[0].strip()
        city = re.sub(r"[\s/]","",city)
        work_years = job_request_spans[2].xpath('.//text()')[0].strip()
        work_years = re.sub(r"[\s/]","",work_years)
        education = job_request_spans[3].xpath('.//text()')[0].strip()
        education = re.sub(r"[\s/]","",education)
        desc = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip()
        company_name = html.xpath("//h2[@class='fl']/text()")[0].strip()
        position = {
            'name':position_name,
            'company_name':company_name,
            'salary':salary,
            'city': city,
            'work_years': work_years,
            'education': education,
            'desc': desc,
        }
        self.position.append(position)
        print(position)
        print('-'*200)

if __name__ == '__main__':
    spider = LagouSpider()
    spider.run()

本文参与腾讯云自媒体分享计划,欢迎正在阅读的你也加入,一起分享。

发表于

我来说两句

0 条评论
登录 后参与评论

相关文章

来自专栏潇涧技术专栏

Builtin Lint Detectors (1)

本文主要介绍的是Lint工具中自带的与Android开发相关的lint检查项,通过查看lint检查项的描述及其代码实现,我发现这里面存在不少应用开发编码的Bes...

10110
来自专栏潇涧技术专栏

Dive into Android Custom View

下面的内容我将结合一个简单的自定义View的例子解释下我们可以如何让自定义View做得更好,这里的自定义View的例子是类似即刻应用中点赞时数字变换效果,代码地...

14430
来自专栏owent

针对Java JIT的优化(转表工具:xresloader)

之前做了一个转Excel表到lua/二进制/json/xml的工具-xresloader。目的一方面是方便策划。另一方面是统一客户端和服务器的转表模式,并且要灵...

9320
来自专栏轻量级微服务

微服务下 Spring Boot Maven 工程依赖关系管理

最基本的 pom.xml 包含工程信息、Spring Boot 父工程、属性配置、依赖包、构建插件

13230
来自专栏蓝天

配置HDFS HttpFS和WebHDFS

默认启动WebHDFS而不会启动HttpFS,而HttpFS需要通过sbin/httpfs.sh来启动。

22830
来自专栏潇涧技术专栏

Android Heroes Reading Notes 2

《Android群英传》读书笔记 (2) 第三章 控件架构与自定义控件详解 + 第四章 ListView使用技巧 + 第五章 Scroll分析

10910
来自专栏zhangdd.com

tomcat安全加固

1. 升级到最新稳定版,这个是老生常谈了。目前Tomcat支持6.0和7.0两个版本。

27120
来自专栏owent

关于firewalld和systemd的一些命令速记

CentOS 7 已经用firewalld替换掉了iptables并用systemd来管理启动服务(之前是chkconfig)。而且下一个Ubuntu的长期支持...

10740
来自专栏潇涧技术专栏

Art of Android Development Reading Notes 6

《Android开发艺术探索》读书笔记 (6) 第6章 Android的Drawable

9020
来自专栏JetpropelledSnake

Python爬虫学习笔记之爬虫基础库

简单来说,Beautiful Soup是python的一个库,最主要的功能是从网页抓取数据。官方解释如下:

13720

扫码关注云+社区

领取腾讯云代金券

年度创作总结 领取年终奖励