代理服务的介绍:
代理的设置:
① urllib的代理设置
from urllib.error import URLError
from urllib.request import ProxyHandler, build_opener
proxy = '127.0.0.1:8888'
#需要认证的代理
#proxy = 'username:password@127.0.0.1:8888'
#使用ProxyHandler设置代理
proxy_handler = ProxyHandler({
'http': 'http://' + proxy,
'https': 'https://' + proxy
})
#传入参数创建Opener对象
opener = build_opener(proxy_handler)
try:
response = opener.open('http://httpbin.org/get')
print(response.read().decode('utf-8'))
except URLError as e:
print(e.reason)
② requests的代理设置
import requests
proxy = '127.0.0.1:8888'
#需要认证的代理
#proxy = 'username:password@127.0.0.1:8888'
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy,
}
try:
response = requests.get('http://httpbin.org/get', proxies=proxies)
print(response.text)
except requests.exceptions.ConnectionError as e:
print('Error', e.args)
③ Selenium的代理使用
from selenium import webdriver
service_args = [
'--proxy=127.0.0.1:9743',
'--proxy-type=http',
#'--proxy-auth=username:password' #带认证代理
]
browser = webdriver.PhantomJS(service_args=service_args)
browser.get('http://httpbin.org/get')
print(browser.page_source)
from selenium import webdriver
proxy = '127.0.0.1:9743'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=http://' + proxy)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get('http://httpbin.org/get')
④ 在Scrapy使用代理
#在Scrapy的Downloader Middleware中间件里
...
def process_request(self, request, spider):
request.meta['proxy'] = 'http://127.0.0.1:9743'
...
免费代理IP的使用
我们可以从互联网中获取免费的代理IP:如:西刺
import requests,random
#定义代理池
proxy_list = [
'182.39.6.245:38634',
'115.210.181.31:34301',
'123.161.152.38:23201',
'222.85.5.187:26675',
'123.161.152.31:23127',
]
# 随机选择一个代理
proxy = random.choice(proxy_list)
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy,
}
try:
response = requests.get('http://httpbin.org/get', proxies=proxies)
print(response.text)
except requests.exceptions.ConnectionError as e:
print('Error', e.args)
收费代理IP的使用
收费代理还是很多的如:
在requests中使用收费代理
import requests
# 从代理服务中获取一个代理IP
proxy = requests.get("http://tvp.daxiangdaili.com/ip/?tid=559775358931681&num=1").text
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy,
}
try:
response = requests.get('http://httpbin.org/get', proxies=proxies)
print(response.text)
except requests.exceptions.ConnectionError as e:
print('Error', e.args)