Python爬取公众号文章并实现关键词分析

import requests
import json
from requests.auth import HTTPProxyAuth

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

# 代理设置（HTTP/HTTPS）
proxies = {
    "http": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}",
    "https": f"http://{proxyUser}:{proxyPass}@{proxyHost}:{proxyPort}"
}

# 请求头
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
    "Cookie": "你的Cookie"  # 替换为你的微信Cookie
}

def get_article_list(offset=0):
    url = "https://mp.weixin.qq.com/mp/profile_ext"
    params = {
        "action": "getmsg",
        "__biz": "公众号的biz参数",  # 替换为目标公众号的biz
        "offset": offset,
        "count": 10,  # 每次请求的文章数量
    }
    try:
        # 使用代理发送请求
        resp = requests.get(
            url,
            params=params,
            headers=headers,
            proxies=proxies,
            timeout=10  # 设置超时时间
        )
        resp.raise_for_status()  # 检查请求是否成功
        data = resp.json()
        return data.get("general_msg_list", [])
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return []

# 测试爬取
articles = get_article_list()
print(json.dumps(articles, indent=2, ensure_ascii=False))

from bs4 import BeautifulSoup

def parse_article_detail(article_url):
    resp = requests.get(article_url, headers=headers)
    soup = BeautifulSoup(resp.text, 'html.parser')
    content = soup.find('div', class_='rich_media_content').get_text()
    return content.strip()

# 示例：解析第一篇文章
first_article_url = articles[0]["app_msg_ext_info"]["content_url"]
article_content = parse_article_detail(first_article_url)
print(article_content[:200])  # 输出前200个字符

import jieba
import jieba.analyse

# 加载停用词
stopwords = set()
with open("stopwords.txt", "r", encoding="utf-8") as f:
    for line in f:
        stopwords.add(line.strip())

# 分词 + 关键词提取
def extract_keywords(text, topK=10):
    words = jieba.cut(text)
    filtered_words = [word for word in words if word not in stopwords and len(word) > 1]
    keywords = jieba.analyse.extract_tags(" ".join(filtered_words), topK=topK)
    return keywords

keywords = extract_keywords(article_content)
print("Top 10 关键词:", keywords)

from wordcloud import WordCloud
import matplotlib.pyplot as plt

def generate_wordcloud(text):
    wc = WordCloud(
        font_path="SimHei.ttf",  # 中文字体
        background_color="white",
        max_words=50,
        width=800,
        height=600
    )
    wc.generate(text)
    plt.imshow(wc, interpolation="bilinear")
    plt.axis("off")
    plt.show()

generate_wordcloud(" ".join(keywords))

import sqlite3

conn = sqlite3.connect("wechat_articles.db")
cursor = conn.cursor()

# 创建表
cursor.execute("""
CREATE TABLE IF NOT EXISTS articles (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    title TEXT,
    publish_time TEXT,
    read_count INTEGER,
    content TEXT,
    keywords TEXT
)
""")

# 插入数据
def save_article(title, publish_time, read_count, content, keywords):
    cursor.execute("""
    INSERT INTO articles (title, publish_time, read_count, content, keywords)
    VALUES (?, ?, ?, ?, ?)
    """, (title, publish_time, read_count, content, ",".join(keywords)))
    conn.commit()

# 示例存储
save_article(
    title="Python爬虫实战",
    publish_time="2023-10-01",
    read_count=5000,
    content=article_content,
    keywords=keywords
)

conn.close()