爬虫程序部署后常见问题整理

import requests

import random

import requests.adapters

# 要访问的目标页面

targetUrlList = [

    "https://",

    "https://",

    "https://",

]

# 代理服务器(产品官网 h.shenlongip.com)

proxyHost = " h.shenlongip.com"

proxyPort = "  "

# 代理验证信息

proxyUser = "username"

proxyPass = "password"

proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {

    "host": proxyHost,

    "port": proxyPort,

    "user": proxyUser,

    "pass": proxyPass,

}

# 设置 http和https访问都是用HTTP代理

proxies = {

    "http": proxyMeta,

    "https": proxyMeta,

}

#  设置IP切换头

tunnel = random.randint(1, 10000)

headers = {"Proxy-Tunnel": str(tunnel)}

class HTTPAdapter(requests.adapters.HTTPAdapter):

    def proxy_headers(self, proxy):

        headers = super(HTTPAdapter, self).proxy_headers(proxy)

        if hasattr(self, 'tunnel'):

            headers['Proxy-Tunnel'] = self.tunnel

        return headers

# 访问三次网站，使用相同的tunnel标志，均能够保持相同的外网IP

for i in range(3):

    s = requests.session()

    a = HTTPAdapter()

    #  设置IP切换头

    a.tunnel = tunnel

    s.mount('https://', a)

    for url in targetUrlList:

        r = s.get(url, proxies=proxies)

        print r.text