网络爬虫 ｜ selenium 爬取动态加载信息

import os
os.chdir(r'/Users/jim/Desktop/代码')
from selenium import webdriver # 导入浏览器驱动模块
browser=webdriver.Chrome() # 会打开浏览器

#根据name
_input=browser.find_element_by_name("q")
#根据css获取，id名
input_second=browser.find_element_by_css_selector("#q")
#根据id获取
input_third=browser.find_element(By.ID,"q")

import time
from selenium import webdriver
browser=webdriver.Chrome()#会打开浏览器
browser.get('https://www.baidu.com/')
_input = browser.find_element_by_id('kw')#找百度的输入框
# print(_input)
_input.send_keys('python')
_input=browser.find_element_by_id('su').click()#回车
time.sleep(10)
browser.close()#关闭浏览器

#获取元素信息，获取属性
from selenium import webdriver
browser=webdriver.Chrome()
url="https://www.zhihu.com/explore"
browser.get(url)
logo=browser.find_element_by_class_name("ZhihuLogoLink")
print(logo)
print(logo.get_attribute("class"))
print(logo.text)
print(logo.id) #id
print(logo.location) #位置
print(logo.tag_name) #标签名
print(logo.size) #大小

import time
from selenium import webdriver
browser=webdriver.Chrome() #打开浏览器
browser.get('https://www.jd.com/?cu=true&utm_source=baidu-pinzhuan&utm_medium=cpc&utm_campaign=t_288551095_baidupinzhuan&utm_term=0f3d30c8dba7459bb52f2eb5eba8ac7d_0_27b1bb5bc1cb4d2faf65fc59cdbfd1cc')
_input=browser.find_element_by_id('key')#
_input.send_keys('羽绒服')
_input.clear()
_input.send_keys('aj')
browser.find_element_by_class_name('button').click()#回车
browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
time.sleep(2)
browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
time.sleep(2)
browser.execute_script('window.scrollTo(0,0)')
#browser.close()#关闭浏览器

from selenium import webdriver
browser = webdriver.Chrome()
browser.get('https://www.zhihu.com/explore')
browser.implicitly_wait(3) # 等待固定时间
_input = browser.find_element_by_class_name('zu-button-more')

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
browser = webdriver.Chrome()
browser.get('https://www.taobao.com/')
wait = WebDriverWait(browser,10) # 最多等待十秒
# 等待到id='q'加载完成
_input = wait.until(EC.presence_of_element_located((By.ID, 'q')))
# 等待到'.btn-search'可以点击
button = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.btn-search')))

from selenium import webdriver
import time
browser=webdriver.Chrome()
browser.get("https://www.taobao.com")
time.sleep(2)
browser.get("https://www.baidu.com")
browser.back()#后退
time.sleep(2)
browser.forward()#前进
time.sleep(3)
browser.refresh()#刷新
time.sleep(3)
browser.close()

from selenium import webdriver
import time
browser=webdriver.Chrome()
browser.get("https://www.zhihu.com/explore")
browser.execute_script("window.open()")
print(browser.window_handles)
browser.switch_to_window(browser.window_handles[1])
browser.get("https://www.taobao.com")
time.sleep(1)
browser.switch_to_window(browser.window_handles[0])
browser.get("https://www.jd.com")
browser.close()
time.sleep(1)
browser.switch_to_window(browser.window_handles[0])
browser.close()

from selenium import webdriver
from selenium.common.exceptions import TimeoutException,NoSuchElementException
browser=webdriver.Chrome()
try:
    browser.get("https://www.zhihu.com/explore")
except TimeoutException:
    print("Time out")
try:
    browser.find_element_by_id("hello")
except NoSuchElementException:
    print("No Element")
finally:
    browser.close()

#常用键的操作
from selenium.webdriver.common.keys import Keys
import time
driver = webdriver.Chrome()
driver.get("http://www.baidu.com")
time.sleep(3)

driver.find_element_by_id('kw').send_keys('selenium')  
#在搜索框中输入"selenium"
time.sleep(3)
driver.find_element_by_id('kw').send_keys(Keys.SPACE)  
#输入空格键
time.sleep(3)
driver.find_element_by_id('kw').send_keys('python')  
#在搜索框中输入"python"
time.sleep(3)
driver.find_element_by_id('kw').send_keys(Keys.CONTROL, 'a')  
#输入Control+a模拟全选
time.sleep(3)
driver.find_element_by_id('kw').send_keys(Keys.CONTROL, 'c')  
#输入Control+c模拟复制
time.sleep(3)
driver.find_element_by_id('kw').send_keys(Keys.CONTROL, 'v')  
#输入Control+v模拟粘贴
time.sleep(3)
driver.find_element_by_id('kw').send_keys(Keys.ENTER)  
#输入回车代替点击搜索按钮
time.sleep(3)
driver.close()

browser=webdriver.Chrome()
url='''
https://passport.jd.com/new/login.aspx?
ReturnUrl=https%3A%2F%2Fwww.jd.com%2F%3Fcu%3
Dtrue%26utm_source%3Dbaidu-pinzhuan%26utm_me
dium%3Dcpc%26utm_campaign%3Dt_288551095_baid
upinzhuan%26utm_term%3D0f3d30c8dba7459bb52f2
eb5eba8ac7d_0_5c595fde6f6743c1a1878bfeaf4f21
5f'''
browser.get(url)
time.sleep(3)
button=browser.find_element_by_link_text('账户登录').click()
browser.find_element_by_id('loginname').send_keys('')#自己的用户名
time.sleep(2)
browser.find_element_by_id("nloginpwd").send_keys('')#密码
time.sleep(2)
browser.find_element_by_id('loginsubmit').click()#登录