Python使用爬虫ip抓取热点新闻

import requests
import random
import time
from bs4 import BeautifulSoup
import csv
from fake_useragent import UserAgent
​
# 配置参数
TARGET_URL = "https://news.sina.com.cn/hotnews/"  # 新浪热点新闻页面
PROXY_POOL = [  # 代理IP列表（需自行替换有效代理）
    {"http": "http://123.456.789.0:8080", "https": "http://123.456.789.0:8080"},
    {"http": "http://111.222.333.444:8888", "https": "http://111.222.333.444:8888"},
]
REQUEST_DELAY = 3  # 请求间隔(秒)
MAX_RETRY = 3     # 最大重试次数
​
class HotNewsCrawler:
    def __init__(self):
        self.ua = UserAgent()
        self.session = requests.Session()
        
    def get_random_headers(self):
        """生成随机请求头"""
        return {
            'User-Agent': self.ua.random,
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Referer': 'https://news.sina.com.cn/'
        }
​
    def get_with_proxy(self, url, proxy=None):
        """使用代理发送请求"""
        try:
            headers = self.get_random_headers()
            response = self.session.get(
                url,
                headers=headers,
                proxies=proxy,
                timeout=10,
                verify=False  # 简单示例关闭SSL验证（生产环境不建议）
            )
            response.raise_for_status()
            if response.status_code == 200:
                return response.text
        except Exception as e:
            print(f"请求失败: {str(e)}")
            return None
​
    def parse_news(self, html):
        """解析新闻页面（示例解析逻辑，需根据实际页面结构调整）"""
        soup = BeautifulSoup(html, 'lxml')
        news_list = []
        
        # 示例选择器（以新浪新闻结构为例）
        for item in soup.select('.news-item'):
            title = item.select_one('h2 a').text.strip()
            link = item.select_one('a')['href']
            news_list.append({"title": title, "link": link})
        
        return news_list
​
    def crawl(self):
        """执行爬取任务"""
        retry_count = 0
        proxy = random.choice(PROXY_POOL) if PROXY_POOL else None
        
        while retry_count < MAX_RETRY:
            html = self.get_with_proxy(TARGET_URL, proxy)
            if html:
                return self.parse_news(html)
            
            retry_count += 1
            print(f"第{retry_count}次重试...")
            time.sleep(REQUEST_DELAY * 2)  # 重试时延长等待
            
        print("达到最大重试次数，爬取失败")
        return []
​
    def save_to_csv(self, data, filename='hot_news.csv'):
        """保存结果到CSV"""
        with open(filename, 'w', newline='', encoding='utf-8-sig') as f:
            writer = csv.DictWriter(f, fieldnames=['title', 'link'])
            writer.writeheader()
            writer.writerows(data)
​
if __name__ == "__main__":
    crawler = HotNewsCrawler()
    
    # 控制请求频率
    time.sleep(REQUEST_DELAY + random.uniform(0, 1))
    
    # 执行爬取
    news_data = crawler.crawl()
    
    if news_data:
        crawler.save_to_csv(news_data)
        print(f"成功抓取{len(news_data)}条热点新闻")
    else:
        print("未获取到数据")

pip install requests beautifulsoup4 fake-useragent csv

def check_proxy(proxy):
    try:
        test_url = "http://httpbin.org/ip"
        response = requests.get(test_url, proxies=proxy, timeout=5)
        return True if response.status_code == 200 else False
    except:
        return False