Python爬虫实例

# 概述

使用爬虫技术，爬取模板之家的模板，下载全站模板。

# 环境

- Python 3.5.4

- selenium：使用非常简洁方便的API让你去使用像Firefox, IE, Chrome, Remote等等这样的Selenium WebDrivers（Selenium web驱动器）

- BeautifulSoup：从HTML或XML文件中提取数据的Python库

# 源码

```

frombs4importBeautifulSoup

fromseleniumimportwebdriver

chrome_options = webdriver.ChromeOptions()

prefs = {

# 'profile.managed_default_content_settings.images': 2,

'download.default_directory':'F:\\soueces'

}

chrome_options.add_experimental_option("prefs", prefs)

browser = webdriver.Chrome(chrome_options=chrome_options)

defget_download_links(url):

browser.get(url)

html = browser.page_source

soup = BeautifulSoup(html,'lxml')

items = soup.select('.thumbItem > li > a')

links = []

foriteminitems:

links.append(link)

returnlinks

defdownload(links):

forlinkinlinks:

browser.get(link)

ele = browser.find_element_by_class_name('btn-down')

ele.click()

defmain():

browser.get(base_url %1)

html = browser.page_source

soup = BeautifulSoup(html,'lxml')

ele = soup.select('.tagsPage > form > font')

pages = ele[1].get_text()# 当前分类公有pages页

i =2

whilei

url = base_url % i

links = get_download_links(url)

download(links)

i = i +1

main()

```