基于bs4+requests的python爬虫伪装 - 草稿

import requests
from bs4 import BeautifulSoup as bs
from fake_useragent import UserAgent
import random

def getSoup(url,encoding="utf-8",**params):
    print(params)
    reponse = requests.get(url,**params)
    reponse.encoding = encoding
    soup = bs(reponse.text,'lxml')
    return soup

def cssFind(movie,cssSelector,nth=1):
    if len(movie.select(cssSelector)) >= nth:
        return movie.select(cssSelector)[nth-1].text.strip()
    else:
        return ''

def getProxyList():
    proxies_url_before = "http://www.66ip.cn/areaindex_2/{}.html"
    proxies_url = proxies_url_before.format(random.randint(1,10))
    soup = getSoup(proxies_url)
    item_list = soup.select("table tr")[2:]
    proxies_list = []
    for item in item_list:
        ipAddress = cssFind(item, "td")
        ipPort = cssFind(item, "td", 2)
        proxies_list.append("http://{}:{}".format(ipAddress, ipPort))
    return proxies_list

def getParams():
    ua = UserAgent()
    ip_list = getProxyList()
    params = dict(
        headers = {'User-Agent': ua.random},
        proxies = {'http': random.choice(ip_list)}
    )
    return params

if __name__ == "__main__":   
    params = getParams()
    soup = getSoup("https://movie.douban.com/top250?start=50",**params)