《手把手带你学爬虫──初级篇》第5课 Selenium WebDriver的用法

pip install selenium

1. 下载 https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-windows.zip
2. 将`phantomjs.exe`所在目录增加到环境变量中

brew install phantomjs

# 导入WebDriver的模块
In [2]: from selenium import webdriver
# 驱动Chrome浏览器，执行后会弹出一个Chrome浏览器
In [3]: browser = webdriver.Chrome()
# 让浏览器访问https://sunjiajia.com
# 执行后弹出的浏览器将访问这个网站
In [4]: browser.get('https://sunjiajia.com')

WebDriverException: Message: 'geckodriver' executable needs to be in PATH.

WebDriverException: Message: 'chromedriver' executable needs to be in PATH. Please see https://sites.google.com/a/chromium.org/chromedriver/home

1. 如果Firefox是32位的，请下载
https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-win32.zip
2. 如果firefox是64位的，请下载
https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-win64.zip
3. 将解压后的geckodriver.exe配置到环境变量中

1. 访问http://chromedriver.chromium.org/downloads，查看与自己Chrome版本对应的chromedriver
2. 下载https://chromedriver.storage.googleapis.com/2.40/chromedriver_win32.zip
3. 将解压后的chromedriver.exe配置到环境变量中

brew install geckodriver
brew cask install chromedriver

下载linux队形的driver，将其放入/usr/bin目录下就可以了。如果没有执行权限，加上执行权限。
sudo chmod +x /usr/bin geckodriver
sudo chmod +x /usr/bin chromedriver

# 导入webdriver
In [7]: from selenium import webdriver
# 导入键盘模拟类 Keys
In [8]: from selenium.webdriver.common.keys import Keys
# 驱动Chrome浏览器
In [9]: browser = webdriver.Chrome()
# 访问京东
In [10]: browser.get('https://www.jd.com')
# 找到id='key'的搜索框页面元素
In [11]: ele_search = browser.find_element_by_id('key')
# 发送关键词 '书包'
In [12]: ele_search.send_keys('书包')
# 发送回车键命令
In [13]: ele_search.send_keys(Keys.ENTER)
# 导入BeautifulSoup库
In [16]: from bs4 import BeautifulSoup
# 拿到标签树对象，浏览器对象的page_source是网页的源码
In [17]: soup = BeautifulSoup(browser.page_source,'lxml')
# 找到title标签
In [18]: tag_title = soup.find('title')
# 打印标签title，证明我们拿到了页面的源码
In [19]: tag_title
Out[19]: <title>书包 - 商品搜索 - 京东</title>

<input name='username' type='text' id='user' />

element = driver.find_element_by_id("user")
element = driver.find_element_by_name("username")
element = driver.find_elements_by_tag_name("input")
element = driver.find_element_by_xpath("//input[@id='user']")

In [3]: browser=webdriver.Chrome()

In [4]: browser.get('http://localhost:63342/codes/lesson05/selenium_demo01.html?_ijt=ai9c4f63mslt0q8a4vb93t7c9l')

In [5]: browser.find_element_by_id('user')
Out[5]: <selenium.webdriver.remote.webelement.WebElement (session="bb336a8045d9536ef7b79e16d5a78637", element="0.6577164491799441-1")>

In [6]: browser.find_element_by_name('username')
Out[6]: <selenium.webdriver.remote.webelement.WebElement (session="bb336a8045d9536ef7b79e16d5a78637", element="0.6577164491799441-1")>

In [7]: browser.find_element_by_tag_name('input')
Out[7]: <selenium.webdriver.remote.webelement.WebElement (session="bb336a8045d9536ef7b79e16d5a78637", element="0.6577164491799441-1")>

In [8]: browser.find_element_by_xpath('//input[@id="user"]')
Out[8]: <selenium.webdriver.remote.webelement.WebElement (session="bb336a8045d9536ef7b79e16d5a78637", element="0.6577164491799441-1")>

In [23]: browser = webdriver.Chrome()
In [24]: browser.get('http://localhost:63342/codes/lesson05/selenium_demo01.html?_ijt=mdvqb7hj3nub56tb9mqm4suktr')

# 获取 input元素
In [25]: ele_input = browser.find_element_by_xpath('//input[@id="user"]')
# 获取类型为submit的input元素
In [26]: ele_submit = browser.find_element_by_xpath('//input[@type="submit"]')
# 向input输入框中输入数据
In [27]: ele_input.send_keys("书包")
# 向input框中再次输入数据，此时，数据直接连接在上面的数据后面
In [28]: ele_input.send_keys("鼠标")
# 清除input中的护具
In [29]: ele_input.clear()
# 输入数据
In [30]: ele_input.send_keys("鼠标")
# 让submit提交按钮进行点击操作
In [31]: ele_submit.click()
# 让浏览器返回到上一页面
In [32]: browser.back()

In [1]: from selenium import webdriver

In [2]: from selenium.webdriver.common.keys import Keys

In [3]: from bs4 import BeautifulSoup

In [4]: browser = webdriver.Chrome()

In [6]: browser.get("http://localhost:63342/codes/lesson05/selenium_demo01.html?_ijt=7c1snu6luuco68a1hc3p3aki85")

In [7]: ele_select = browser.find_element_by_id('province')
    
In [9]: eles_option = browser.find_elements_by_tag_name('option')

In [10]: for option in eles_option:
    ...:     print(option.get_attribute('value'))
    ...:     option.click()
    ...:
bj
sh
gz
zz

In [11]: from selenium.webdriver.support.ui import Select

In [12]: ele_select = Select(browser.find_element_by_name('province'))

In [13]: ele_select.select_by_index(0)

In [17]: ele_select.select_by_index(1)

In [18]: ele_select.select_by_value('zz')

In [19]: ele_select.select_by_visible_text('北京')

In [45]: browser = webdriver.Chrome()

In [46]: browser.get('http://localhost:63342/codes/lesson05/selenium_demo01.html?_ijt=8838lsi704td3g3va8uoqc6bhb')

# 用Select方法得到select对象
In [47]: ele_select = Select(browser.find_element_by_id('fruits'))
# 是否是多选select
In [48]: ele_select.is_multiple
Out[48]: True
# 按照索引值进行选中
In [49]: ele_select.select_by_index(0)

In [50]: ele_select.select_by_index(1)

In [51]: ele_select.select_by_index(2)
# 按照value值选中
In [53]: ele_select.select_by_value('pg')
# 被选中的option元素
In [55]: ele_select.all_selected_options
Out[55]:
[<selenium.webdriver.remote.webelement.WebElement (session="8b62401cbaeed777430915d8d2b4d834", element="0.9216159265436044-2")>,
 <selenium.webdriver.remote.webelement.WebElement (session="8b62401cbaeed777430915d8d2b4d834", element="0.9216159265436044-3")>,
 <selenium.webdriver.remote.webelement.WebElement (session="8b62401cbaeed777430915d8d2b4d834", element="0.9216159265436044-4")>,
 <selenium.webdriver.remote.webelement.WebElement (session="8b62401cbaeed777430915d8d2b4d834", element="0.9216159265436044-5")>]

# 循环遍历被选中的元素对象，打印其value属性的值
In [56]: for option in ele_select.all_selected_options:
    ...:    print(option.get_attribute('value'))
    ...:
xj
xg
xhs
pg

# 根据索引值反选
In [57]: ele_select.deselect_by_index(0)
# 反选全部
In [58]: ele_select.deselect_all()

In [60]: browser = webdriver.Chrome()
In [62]: browser.get('https://www.baidu.com')

# 获取百度的搜索框
In [63]: ele_input = browser.find_element_by_name('wd')
# 输入内容
In [64]: ele_input.send_keys('sunjiajia.com')
# 获取按钮
In [65]: ele_submit = browser.find_element_by_id('su')
# 点击按钮
In [66]: ele_submit.click()
# 获取下一页元素
In [67]: ele_next = browser.find_element_by_class_name('n')
# 点击下一页元素
In [68]: ele_next.click()
# 再次点击下一页元素，发现报错，因为当前页面的内容已经改变，需要重新获取下一页元素
In [69]: ele_next.click()
# 重新获取下一页元素
In [70]: ele_next = browser.find_element_by_class_name('n')
# 点击元素，我们发现竟然回到了第一页，那是因为，当前页非第1页时，页面上出现了上一页元素，class属性值也为n，因此，这时得到的元素为上一页元素
In [71]: ele_next.click()
# 获取class值为n的所有元素，取最后一个，就是下一页元素
In [74]: ele_next = browser.find_elements_by_class_name('n')[-1]
# 点击下一页元素
In [75]: ele_next.click()
# 再次获取下一页元素
In [77]: ele_next = browser.find_elements_by_class_name('n')[-1]
# 再次点击下一页元素
In [78]: ele_next.click()
# 操作浏览器后退
In [79]: browser.back()
# 操作浏览器前进
In [80]: browser.forward()

In [100]: browser.get('https://www.baidu.com')

# 获取所有Cookie
In [101]: browser.get_cookies()
Out[101]:
[{'domain': '.baidu.com',
  'httpOnly': False,
  'name': 'H_PS_PSSID',
  'path': '/',
  'secure': False,
  'value': '1426_21117_26350_26433_22075'},
 {'domain': '.baidu.com',
  'expiry': 3677830659.441811,
  'httpOnly': False,
  'name': 'BIDUPSID',
  'path': '/',
  'secure': False,
  'value': 'C914BFBE505CBD0F56D8C9E95B71AF50'},
 {'domain': '.baidu.com',
  'expiry': 3677830659.441852,
  'httpOnly': False,
  'name': 'PSTM',
  'path': '/',
  'secure': False,
  'value': '1530347012'},
 {'domain': '.baidu.com',
  'expiry': 1530433417.192771,
  'httpOnly': False,
  'name': 'BDORZ',
  'path': '/',
  'secure': False,
  'value': 'B490B5EBF6F3CD402E515D22BCDA1598'},
 {'domain': 'www.baidu.com',
  'expiry': 1531211072,
  'httpOnly': False,
  'name': 'BD_UPN',
  'path': '/',
  'secure': False,
  'value': '123253'},
 {'domain': 'www.baidu.com',
  'httpOnly': False,
  'name': 'BD_HOME',
  'path': '/',
  'secure': False,
  'value': '0'},
 {'domain': '.baidu.com',
  'expiry': 3677830659.441722,
  'httpOnly': False,
  'name': 'BAIDUID',
  'path': '/',
  'secure': False,
  'value': 'C914BFBE505CBD0F56D8C9E95B71AF50:FG=1'}]
# 添加Cookie
In [102]: browser.add_cookie(cookie_dict=browser.get_cookies()[-1])

In [119]: browser = webdriver.Chrome()

In [120]: browser.get('https://www.jd.com')

In [121]: ele_search = browser.find_element_by_id('key')

In [122]: ele_submit = browser.find_element_by_class_name('button')

In [123]: ele_search.send_keys('书包')

In [124]: ele_submit.click()

In [125]: from bs4 import BeautifulSoup

In [126]: soup = BeautifulSoup(browser.page_source,'lxml')
# 此时，页面上商品信息只有30个
In [127]: len(soup.findAll('li',class_='gl-item'))
Out[127]: 30
# 将滚动条滚动到底部
In [128]: browser.execute_script('window.scrollTo(0, document.body.scrollHeight)')
# 重新生成标签树对象
In [129]: soup = BeautifulSoup(browser.page_source,'lxml')
# 此时，页面上的商品信息有60个
In [130]: len(soup.findAll('li',class_='gl-item'))
Out[130]: 60

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("http://somedomain/url_that_delays_loading")
try:
    element = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "myDynamicElement"))
    )
finally:
    driver.quit()

from selenium import webdriver

driver = webdriver.Firefox()
driver.implicitly_wait(10) # seconds
driver.get("http://somedomain/url_that_delays_loading")
myDynamicElement = driver.find_element_by_id("myDynamicElement")

In [143]: browser = webdriver.PhantomJS()
/Users/sunjiajia/Works/pyenv/crawler/lib/python3.6/site-packages/selenium/webdriver/phantomjs/webdriver.py:49: UserWarning: Selenium support for PhantomJS has been deprecated, please use headless versions of Chrome or Firefox instead
  warnings.warn('Selenium support for PhantomJS has been deprecated, please use headless '

In [146]: browser.get('https://www.baidu.com')

In [148]: browser.page_source[:1000]
Out[148]: '<!DOCTYPE html><!--STATUS OK--><html><head>\n    \n    <meta http-equiv="content-type" content="text/html;charset=utf-8">\n    <meta http-equiv="X-UA-Compatible" content="IE=Edge">\n\t<meta content="always" name="referrer">\n    <meta name="theme-color" content="#2932e1">\n    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">\n    <link rel="search" type="application/opensearchdescription+xml" href="/content-search.xml" title="百度搜索">\n    <link rel="icon" sizes="any" mask="" href="//www.baidu.com/img/baidu_85beaf5496f291521eb75ba38eacbd87.svg">\n\t\n\t\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com">\n\t<link rel="dns-prefetch" href="//t1.baidu.com">\n\t<link rel="dns-prefetch" href="//t2.baidu.com">\n\t<link rel="dns-prefetch" href="//t3.baidu.com">\n\t<link rel="dns-prefetch" href="//t10.baidu.com">\n\t<link rel="dns-prefetch" href="//t11.baidu.com">\n\t<link rel="dns-prefetch" href="//t12.baidu.com">\n\t<link rel="dns-prefetch" href="//b1.bdstatic.com">\n    \n    <title>百度一下，你就知道</title>\n  '

In [1]: from selenium import webdriver

In [2]: from selenium.webdriver.chrome.options import Options as ChromeOptions

In [3]: chrome_options = ChromeOptions()

In [4]: chrome_options.add_argument('--headless')

In [5]: chrome_options.add_argument('--disable-gpu')

In [6]: browser = webdriver.Chrome(chrome_options=chrome_options)

In [7]: browser.get('https://www.baidu.com')

In [8]: browser.page_source[:1000]
Out[8]: '<!DOCTYPE html><!--STATUS OK--><html xmlns="http://www.w3.org/1999/xhtml"><head>\n    \n    <meta http-equiv="content-type" content="text/html;charset=utf-8" />\n    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />\n\t<meta content="always" name="referrer" />\n    <meta name="theme-color" content="#2932e1" />\n    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />\n    <link rel="search" type="application/opensearchdescription+xml" href="/content-search.xml" title="百度搜索" />\n    <link rel="icon" sizes="any" mask="" href="//www.baidu.com/img/baidu_85beaf5496f291521eb75ba38eacbd87.svg" />\n\t\n\t\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com" />\n\t<link rel="dns-prefetch" href="//t1.baidu.com" />\n\t<link rel="dns-prefetch" href="//t2.baidu.com" />\n\t<link rel="dns-prefetch" href="//t3.baidu.com" />\n\t<link rel="dns-prefetch" href="//t10.baidu.com" />\n\t<link rel="dns-prefetch" href="//t11.baidu.com" />\n\t<link rel="dns-prefetch" href="//t12.baidu.com" />\n\t<link rel="dns-prefetc'

In [1]: from selenium import webdriver

In [2]: from selenium.webdriver.firefox.options import Options as FFOptions

In [3]: ff_options = FFOptions()

In [4]: ff_options.add_argument('--headless')

In [5]: browser = webdriver.Firefox(firefox_options=ff_options)
    
In [7]: browser.get("https://www.baidu.com")

In [9]: browser.page_source[:1000]
Out[9]: '<html><head>\n    \n    <meta http-equiv="content-type" content="text/html;charset=utf-8">\n    <meta http-equiv="X-UA-Compatible" content="IE=Edge">\n\t<meta content="always" name="referrer">\n    <meta name="theme-color" content="#2932e1">\n    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">\n    <link rel="search" type="application/opensearchdescription+xml" href="/content-search.xml" title="百度搜索">\n    <link rel="icon" sizes="any" mask="" href="//www.baidu.com/img/baidu_85beaf5496f291521eb75ba38eacbd87.svg">\n\t\n\t\n\t<link rel="dns-prefetch" href="//s1.bdstatic.com">\n\t<link rel="dns-prefetch" href="//t1.baidu.com">\n\t<link rel="dns-prefetch" href="//t2.baidu.com">\n\t<link rel="dns-prefetch" href="//t3.baidu.com">\n\t<link rel="dns-prefetch" href="//t10.baidu.com">\n\t<link rel="dns-prefetch" href="//t11.baidu.com">\n\t<link rel="dns-prefetch" href="//t12.baidu.com">\n\t<link rel="dns-prefetch" href="//b1.bdstatic.com">\n    \n    <title>百度一下，你就知道</title>\n    \n\n<style id="css_index" index'

#访问xxxx 网站
driver.get("http://www.xxxx.com/")
# 将用户名密码写入浏览器cookie
driver.add_cookie({'name':'username', 'value':'gitopen'})
driver.add_cookie({'name':'password', 'value':'123456'})
# 再次访问xxxx 网站，将会自动登录
driver.get("http://www.xxxx.com/")

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import time
import json

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--diable-gpu')

browser = webdriver.Chrome(chrome_options=chrome_options)
browser.get("https://hr.tencent.com/position.php?&start=0#a")

result = []
while True:
    job = {}
    try:
        element = WebDriverWait(browser, 10).until(
            EC.presence_of_all_elements_located((By.ID, 'next'))
        )
    except Exception as e:
        print(str(e))
    browser.execute_script('window.scrollTo(0, document.body.scrollHeight)')

    # 解析内容
    print("## 正在爬取连接：{}".format(browser.current_url))
    soup = BeautifulSoup(browser.page_source, 'lxml')
    tag_table = soup.find('table')
    tr_even = tag_table.find_all('tr', {'class': 'even'})
    tr_odd = tag_table.find_all('tr', {'class': 'odd'})

    for tr in tr_even + tr_odd:
        tds = tr.find_all('td')
        job['url'] = "https://hr.tencent.com/" + tds[0].a.attrs['href']
        job['title'] = tds[0].a.string
        job['level'] = tds[1].string if tds[1].string is not None else ""
        job['num'] = tds[2].string
        job['location'] = tds[3].string
        job['date'] = tds[4].string
    # 将每次爬取到的信息放入最终结果字典中
    result.append(job)

    # 拿到下一页tag
    a_next = soup.find(id='next')
    # 判断下一页能否点击，如果不可以，说明当前页是最后一页，退出循环；如果能够点击，那么点击它，继续循环
    if 'class' in a_next.attrs.keys() and a_next.attrs['class'][0] == 'noactive':
        break
    else:
        ele_next = browser.find_element_by_id('next')
        ele_next.click()

# 将结果保存到文件中，以json格式
localtime = "-".join(time.asctime(time.localtime(time.time())).split(' '))
with open("job_" + localtime + '.json', 'w') as filename:
    filename.write(json.dumps(result, ensure_ascii=False))
print("### 爬取完毕")