域名列表爬虫

import requests
from bs4 import BeautifulSoup
import random
import time

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}

proxies = [
    {'http': 'http://proxy1.example.com:8080'},
    {'http': 'http://proxy2.example.com:8080'},
    # 添加更多代理IP
]

def get_domains(url):
    proxy = random.choice(proxies)
    response = requests.get(url, headers=headers, proxies=proxy)
    soup = BeautifulSoup(response.text, 'html.parser')
    domains = [a['href'] for a in soup.find_all('a', href=True) if 'http' in a['href']]
    return domains

urls = ['http://example.com/page1', 'http://example.com/page2']
for url in urls:
    domains = get_domains(url)
    print(domains)
    time.sleep(random.uniform(1, 3))  # 随机等待1到3秒