Selenium+代理爬取需要模拟用户交互的网站

    import os
    import time
    import zipfile

    from selenium import webdriver
    from selenium.common.exceptions import TimeoutException
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.support.ui import WebDriverWait


    class GenCookies(object):
        # 随机useragent
        USER_AGENT = open('useragents.txt').readlines()


        # 代理服务器(产品官网 www.16yun.cn)
        PROXY_HOST = 't.16yun.cn'  #  proxy or host
        PROXY_PORT = 31111  # port
        PROXY_USER = 'USERNAME'  # username
        PROXY_PASS = 'PASSWORD'  # password

        @classmethod
        def get_chromedriver(cls, use_proxy=False, user_agent=None):
            manifest_json = """
            {
                "version": "1.0.0",
                "manifest_version": 2,
                "name": "Chrome Proxy",
                "permissions": [
                    "proxy",
                    "tabs",
                    "unlimitedStorage",
                    "storage",
                    "<all_urls>",
                    "webRequest",
                    "webRequestBlocking"
                ],
                "background": {
                    "scripts": ["background.js"]
                },
                "minimum_chrome_version":"22.0.0"
            }
            """

            background_js = """
            var config = {
                    mode: "fixed_servers",
                    rules: {
                    singleProxy: {
                        scheme: "http",
                        host: "%s",
                        port: parseInt(%s)
                    },
                    bypassList: ["localhost"]
                    }
                };

            chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

            function callbackFn(details) {
                return {
                    authCredentials: {
                        username: "%s",
                        password: "%s"
                    }
                };
            }

            chrome.webRequest.onAuthRequired.addListener(
                        callbackFn,
                        {urls: ["<all_urls>"]},
                        ['blocking']
            );
            """ % (cls.PROXY_HOST, cls.PROXY_PORT, cls.PROXY_USER, cls.PROXY_PASS)
            path = os.path.dirname(os.path.abspath(__file__))
            chrome_options = webdriver.ChromeOptions()

            # 关闭webdriver的一些标志
            # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])        


            if use_proxy:
                pluginfile = 'proxy_auth_plugin.zip'

                with zipfile.ZipFile(pluginfile, 'w') as zp:
                    zp.writestr("manifest.json", manifest_json)
                    zp.writestr("background.js", background_js)
                chrome_options.add_extension(pluginfile)
            if user_agent:
                chrome_options.add_argument('--user-agent=%s' % user_agent)
            driver = webdriver.Chrome(
                os.path.join(path, 'chromedriver'),
                chrome_options=chrome_options)

            # 修改webdriver get属性
            # script = '''
            # Object.defineProperty(navigator, 'webdriver', {
            # get: () => undefined
            # })
            # '''
            # driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": script})

            return driver

        def __init__(self, username, password):        
            # 登录example网站
            self.url = 'https://passport.example.cn/signin/login?entry=example&r=https://m.example.cn/'
            self.browser = self.get_chromedriver(use_proxy=True, user_agent=self.USER_AGENT)
            self.wait = WebDriverWait(self.browser, 20)
            self.username = username
            self.password = password

        def open(self):
            """
            打开网页输入用户名密码并点击
            :return: None
            """
            self.browser.delete_all_cookies()
            self.browser.get(self.url)
            username = self.wait.until(EC.presence_of_element_located((By.ID, 'loginName')))
            password = self.wait.until(EC.presence_of_element_located((By.ID, 'loginPassword')))
            submit = self.wait.until(EC.element_to_be_clickable((By.ID, 'loginAction')))
            username.send_keys(self.username)
            password.send_keys(self.password)
            time.sleep(1)
            submit.click()

        def password_error(self):
            """
            判断是否密码错误
            :return:
            """
            try:
                return WebDriverWait(self.browser, 5).until(
                    EC.text_to_be_present_in_element((By.ID, 'errorMsg'), '用户名或密码错误'))
            except TimeoutException:
                return False

        def get_cookies(self):
            """
            获取Cookies
            :return:
            """
            return self.browser.get_cookies()

        def main(self):
            """
            入口
            :return:
            """
            self.open()
            if self.password_error():
                return {
                    'status': 2,
                    'content': '用户名或密码错误'
                }            

            cookies = self.get_cookies()
            return {
                'status': 1,
                'content': cookies
            }


    if __name__ == '__main__':
        result = GenCookies(
            username='180000000',
            password='16yun',
        ).main()
        print(result)