新闻热点一目了然：Python爬虫数据可视化

import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import re

# 代理配置信息
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

def crawl_news(url):
    """
    爬取新闻页面的标题和链接
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    
    # 构建代理信息
    proxyMeta = f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
    proxies = {
        "http": proxyMeta,
        "https": proxyMeta
    }
    
    try:
        response = requests.get(url, headers=headers, proxies=proxies, timeout=10)
        response.encoding = 'utf-8' # 根据目标网站调整编码
        soup = BeautifulSoup(response.text, 'html.parser')
        
        news_list = []
        # 假设新闻条目在 <div class='news-item'> 的 <a> 标签里
        # !!! 此选择器需要根据目标网站的实际HTML结构进行修改 !!!
        news_items = soup.find_all('div', class_='news-item')
        
        for item in news_items:
            a_tag = item.find('a')
            if a_tag:
                title = a_tag.get_text().strip()
                link = a_tag.get('href')
                # 处理相对链接
                if link and not link.startswith('http'):
                    link = requests.compat.urljoin(url, link)
                news_list.append({'title': title, 'link': link})
        
        return news_list
    except Exception as e:
        print(f"爬取过程中发生错误: {e}")
        return []

# 示例：爬取多个页面（这里以2页为例）
all_news = []
base_url = "https://example-news-site.com/page/{}" # 请替换为真实URL

for page in range(1, 3):
    url = base_url.format(page)
    print(f"正在爬取: {url}")
    news_on_page = crawl_news(url)
    all_news.extend(news_on_page)
    time.sleep(1) # 礼貌性延时，避免对服务器造成压力

# 转换为DataFrame方便处理
df_news = pd.DataFrame(all_news)
print(f"共爬取到 {len(df_news)} 条新闻。")
print(df_news.head())

import jieba
from collections import Counter

# 定义停用词列表（这里是一个简单示例，实际项目中需要更全面的停用词表）
stopwords = set(['的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这个', '那个'])

def extract_keywords_from_titles(titles):
    """
    从标题列表中提取关键词
    """
    all_words = []
    for title in titles:
        # 使用jieba进行精确模式分词
        words = jieba.cut(title)
        # 过滤：只保留非停用词、长度大于1的词（去除单字），并且是中文或数字字母
        filtered_words = [
            word for word in words 
            if word not in stopwords 
            and len(word) > 1 
            and re.match("^[\u4e00-\u9fa5a-zA-Z0-9]+$", word)
        ]
        all_words.extend(filtered_words)
    
    # 统计词频
    word_counts = Counter(all_words)
    return word_counts

# 从DataFrame的'title'列提取关键词
titles = df_news['title'].tolist()
keyword_counts = extract_keywords_from_titles(titles)

# 查看最常见的20个关键词
top_keywords = keyword_counts.most_common(20)
print("热点关键词TOP20:")
for word, count in top_keywords:
    print(f"{word}: {count}")

from pyecharts import options as opts
from pyecharts.charts import WordCloud

# 准备词云数据，格式为 [(word1, count1), (word2, count2), ...]
wordcloud_data = [(word, count) for word, count in top_keywords]

# 创建词云对象
wordcloud = (
    WordCloud()
    .add(
        series_name="新闻热点",
        data_pair=wordcloud_data,
        word_size_range=[20, 80], # 词语字体大小范围
        shape="circle", # 词云形状，可选 'circle', 'cardioid', 'diamond'等
    )
    .set_global_opts(
        title_opts=opts.TitleOpts(
            title="新闻热点词云",
            title_textstyle_opts=opts.TextStyleOpts(font_size=23)
        ),
        tooltip_opts=opts.TooltipOpts(is_show=True),
    )
)

# 渲染词云到HTML文件
wordcloud.render("news_word_cloud.html")
print("词云图已生成，请打开 'news_word_cloud.html' 文件查看。")

from pyecharts.charts import Bar

# 分离出Top 10的关键词和对应的数量
top_10_words = [item[0] for item in top_keywords[:10]]
top_10_counts = [item[1] for item in top_keywords[:10]]

# 创建条形图对象
bar = (
    Bar()
    .add_xaxis(top_10_words)
    .add_yaxis("出现频次", top_10_counts)
    .set_global_opts(
        title_opts=opts.TitleOpts(title="新闻热点关键词TOP10"),
        xaxis_opts=opts.AxisOpts(axislabel_opts=opts.LabelOpts(rotate=-15)), # 倾斜X轴标签避免重叠
        yaxis_opts=opts.AxisOpts(name="频次"),
        tooltip_opts=opts.TooltipOpts(trigger="axis"),
    )
    .set_series_opts(
        label_opts=opts.LabelOpts(is_show=True) # 在柱子上显示数值
    )
)

# 渲染条形图到HTML文件
bar.render("news_top_keywords.html")
print("关键词条形图已生成，请打开 'news_top_keywords.html' 文件查看。")