Python爬取懂车帝/汽车之家评论并做竞品分析

import requests
from bs4 import BeautifulSoup
import pandas as pd

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}

def get_autohome_comments(car_id, page=1):
    url = f"https://club.autohome.com.cn/bbs/thread-c-{car_id}-{page}.html"
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    comments = soup.find_all('div', class_='tz-paragraph')
    return [comment.get_text().strip() for comment in comments]

# 示例：爬取某车型的评论（car_id需替换）
comments = get_autohome_comments("1234", 1)  # 1234是车型ID
print(comments[:3])  # 输出前3条评论

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
import time

service = Service("path/to/chromedriver")  # 需下载对应ChromeDriver
driver = webdriver.Chrome(service=service)

def get_dongchedi_comments(car_id):
    url = f"https://www.dongchedi.com/community/{car_id}"
    driver.get(url)
    time.sleep(3)  # 等待加载
    
    # 模拟滚动加载更多评论
    for _ in range(3):  # 滚动3次
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
    
    comments = driver.find_elements(By.CLASS_NAME, "comment-content")
    return [comment.text for comment in comments]

# 示例：爬取懂车帝某车型评论
comments = get_dongchedi_comments("1001")  # 1001是车型ID
print(comments[:3])
driver.quit()

import pandas as pd

data = {
    "source": ["autohome"] * len(autohome_comments) + ["dongchedi"] * len(dongchedi_comments),
    "comment": autohome_comments + dongchedi_comments
}

df = pd.DataFrame(data)
df.to_csv("car_comments.csv", index=False)

import pymysql

conn = pymysql.connect(
    host="localhost",
    user="root",
    password="your_password",
    database="car_analysis"
)

cursor = conn.cursor()
cursor.execute("""
    CREATE TABLE IF NOT EXISTS comments (
        id INT AUTO_INCREMENT PRIMARY KEY,
        source VARCHAR(20),
        comment TEXT
    )
""")

# 插入数据
for index, row in df.iterrows():
    cursor.execute("INSERT INTO comments (source, comment) VALUES (%s, %s)", (row["source"], row["comment"]))

conn.commit()
conn.close()

import jieba
from wordcloud import WordCloud
import matplotlib.pyplot as plt

text = " ".join(df["comment"])
words = jieba.lcut(text)
word_freq = pd.Series(words).value_counts().head(20)

# 生成词云
wordcloud = WordCloud(font_path="simhei.ttf", background_color="white").generate(" ".join(words))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

from snownlp import SnowNLP

def get_sentiment(text):
    return SnowNLP(text).sentiments

df["sentiment"] = df["comment"].apply(get_sentiment)

# 按来源（懂车帝/汽车之家）分析情感倾向
sentiment_by_source = df.groupby("source")["sentiment"].mean()
print(sentiment_by_source)

import seaborn as sns

# 绘制情感分布
sns.boxplot(x="source", y="sentiment", data=df)
plt.title("Sentiment Analysis: Autohome vs Dongchedi")
plt.show()