基于多模态内容理解的智能SEO优化系统

┌─────────────────────────────────────────────────────────┐
│                   智能SEO优化系统                         │
├─────────────────────────────────────────────────────────┤
│                                                         │
│  ┌──────────┐    ┌──────────────┐    ┌──────────────┐  │
│  │ 内容输入  │───│ 文本预处理    │───│ 特征提取器    │  │
│  │ (Input)  │    │ (Preprocess) │    │ (Extractor)  │  │
│  └──────────┘    └──────────────┘    └──────┬───────┘  │
│                                             │          │
│                          ┌──────────────────┼───────┐  │
│                          ▼                  ▼       ▼  │
│                   ┌───────────┐  ┌──────────┐ ┌──────┐│
│                   │关键词挖掘  │  │TF-IDF分析│ │语义  ││
│                   │(Keyword)  │  │ (TF-IDF) │ │相似度││
│                   └─────┬─────┘  └────┬─────┘ └──┬───┘│
│                         │             │           │    │
│                         └──────┬──────┴───────────┘    │
│                                ▼                        │
│                       ┌────────────────┐                │
│                       │   内容评分器    │                │
│                       │  (Scorer)      │                │
│                       └───────┬────────┘                │
│                               │                         │
│                    ┌──────────┼──────────┐              │
│                    ▼          ▼          ▼              │
│            ┌──────────┐ ┌────────┐ ┌──────────┐       │
│            │内链优化   │ │竞品对比 │ │排名追踪   │       │
│            │(Internal) │ │(Compete│ │(Ranking) │       │
│            └──────────┘ └────────┘ └──────────┘       │
│                    │          │          │              │
│                    └──────────┼──────────┘              │
│                               ▼                         │
│                       ┌────────────────┐                │
│                       │  优化报告生成    │                │
│                       │  (Reporter)    │                │
│                       └────────────────┘                │
└─────────────────────────────────────────────────────────┘

"""
smart_seo_optimizer.py - 基于多模态内容理解的智能SEO优化系统
技术栈: Python / math / collections / typing / re / json / hashlib
场景: 关键词挖掘、TF-IDF分析、语义相似度、内容评分、内链优化、竞品对比
"""
import os
import re
import json
import math
import uuid
import hashlib
import logging
from typing import List, Dict, Optional, Any, Tuple, Set
from dataclasses import dataclass, field, asdict
from enum import Enum
from datetime import datetime
from collections import defaultdict, Counter

# ==================== 模块1: 数据模型定义 ====================

class ContentType(Enum): 17022.baike.tongsou.com 
    """内容类型"""
    ARTICLE = "article"
    PRODUCT = "product"
    LANDING_PAGE = "landing_page"
    BLOG = "blog"
    FAQ = "faq"


class KeywordDifficulty(Enum):
    """关键词难度等级"""
    EASY = "easy"
    MEDIUM = "medium"
    HARD = "hard"
    VERY_HARD = "very_hard"


class OptimizationPriority(Enum):
    """优化优先级"""
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"


@dataclass
class Keyword: 17026.baike.tongsou.com 
    """关键词模型"""
    keyword_id: str
    text: str
    search_volume: int  # 月搜索量
    competition: float  # 竞争度 0-1
    cpc: float  # 点击成本
    difficulty: KeywordDifficulty
    related_keywords: List[str] = field(default_factory=list)
    intent: str = ""  # informational / navigational / commercial / transactional
    score: float = 0.0  # 综合评分


@dataclass
class TFIDFResult: 17117.baike.tongsou.com 
    """TF-IDF分析结果"""
    term: str
    tf: float
    idf: float
    tfidf: float
    count: int
    density: float


@dataclass
class ContentScore: 17121.baike.tongsou.com 
    """内容评分"""
    content_id: str
    overall_score: float  # 0-100
    readability_score: float
    keyword_coverage_score: float
    structure_score: float
    length_score: float
    freshness_score: float
    issues: List[Dict[str, Any]] = field(default_factory=list)
    suggestions: List[str] = field(default_factory=list)


@dataclass
class SemanticSimilarity: 17122.baike.tongsou.com 
    """语义相似度结果"""
    source_text: str
    target_text: str
    similarity_score: float  # 0-1
    common_terms: List[str]
    missing_terms: List[str]


@dataclass
class InternalLinkSuggestion:
    """内链优化建议"""
    suggestion_id: str
    source_content_id: str
    target_content_id: str
    anchor_text: str
    relevance_score: float
    priority: OptimizationPriority
    reason: str


@dataclass
class CompetitorAnalysis: 17129.baike.tongsou.com 
    """竞品分析结果"""
    competitor_url: str
    title: str
    word_count: int
    keyword_count: int
    heading_count: int
    image_count: int
    link_count: int
    avg_sentence_length: float
    keyword_density: Dict[str, float]
    strengths: List[str]
    weaknesses: List[str]


@dataclass
class RankingRecord: 17135.baike.tongsou.com 
    """排名记录"""
    record_id: str
    keyword: str
    url: str
    position: int
    search_engine: str
    location: str
    timestamp: str
    previous_position: int = 0
    change: int = 0


# ==================== 模块2: 文本预处理 ====================

class TextPreprocessor: 17137.baike.tongsou.com 
    """
    文本预处理器
    负责清洗、分词、标准化文本
    """

    def __init__(self): 17147.baike.tongsou.com 
        # 常见停用词表（简化版）
        self.stopwords = set([
            "的", "了", "在", "是", "我", "有", "和", "就",
            "不", "人", "都", "一", "一个", "上", "也", "很",
            "到", "说", "要", "去", "你", "会", "着", "没有",
            "看", "好", "自己", "这", "他", "她", "它", "们",
            "那", "些", "什么", "怎么", "可以", "被", "把",
            "the", "is", "in", "it", "of", "and", "to", "a",
            "an", "for", "on", "with", "as", "at", "by", "from",
            "this", "that", "was", "are", "be", "has", "have",
            "but", "not", "you", "all", "can", "her", "was",
            "one", "our", "out", "day", "get", "has", "him",
            "his", "how", "its", "may", "new", "now", "old",
            "see", "way", "who", "did", "get", "let", "say",
            "she", "too", "use",
        ])

    def clean_text(self, text: str) -> str: 17148.baike.tongsou.com 
        """清洗文本：去除多余空白、特殊字符等"""
        # 去除HTML标签
        text = re.sub(r'<[^>]+>', '', text)
        # 去除多余空白
        text = re.sub(r'\s+', ' ', text).strip(17152.baike.tongsou.com)
        # 去除特殊字符（保留中英文、数字、标点）
        text = re.sub(r'[^\w\s\u4e00-\u9fff.,!?;:]', '', text)
        return text

    def tokenize(self, text: str, remove_stopwords: bool = True) -> List[str]:
        """
        简单分词（基于空格和常见分隔符）
        注意：生产环境应使用jieba等中文分词库
        """
        # 英文分词
        tokens = re.findall(r'[a-zA-Z]+|\u4e00-\u9fff', text.lower())
        if remove_stopwords:
            tokens = [t for t in tokens if t not in self.stopwords and len(t) > 1]
        return tokens

    def extract_sentences(self, text: str) -> List[str]: 17153.baike.tongsou.com 
        """提取句子"""
        sentences = re.split(r'[.!?。！？]', text)
        return [s.strip() for s in sentences if s.strip()]

    def count_words(self, text: str) -> int: 17154.baike.tongsou.com 
        """统计词数"""
        tokens = self.tokenize(text, remove_stopwords=False)
        return len(tokens)

    def count_characters(self, text: str) -> int: 17162.baike.tongsou.com 
        """统计字符数"""
        return len(text)

    def avg_sentence_length(self, text: str) -> float: 17157.baike.tongsou.com 
        """计算平均句长"""
        sentences = self.extract_sentences(text)
        if not sentences:
            return 0.0
        total_words = sum(len(self.tokenize(s, remove_stopwords=False)) for s in sentences)
        return total_words / len(sentences)


# ==================== 模块3: 关键词挖掘引擎 ====================

class KeywordMiner: 17168.baike.tongsou.com 
    """
    关键词挖掘引擎
    从文本中提取关键词，并计算综合评分
    """

    def __init__(self, preprocessor: TextPreprocessor):
        self.preprocessor = preprocessor
        self.keywords: Dict[str, Keyword] = {}

    def extract_keywords(self, text: str, top_n: int = 20) -> List[Keyword]:
        """
        从文本中提取关键词
        使用词频+词长加权算法
        """
        tokens = self.preprocessor.tokenize(text)
        if not tokens: 17171.baike.tongsou.com 
            return []

        # 统计词频
        freq = Counter(tokens)
        total_tokens = len(tokens)

        keywords = []
        for term, count in freq.most_common(top_n * 2):
            # 计算综合评分：词频 * 词长权重
            tf = count / total_tokens
            length_weight = min(len(term) / 10, 1.0)
            score = tf * (1 + length_weight) * 100

            difficulty = self._estimate_difficulty(count, total_tokens)
            kw = Keyword(
                keyword_id=hashlib.md5(term.encode()).hexdigest()[:12],
                text=term,
                search_volume=self._estimate_search_volume(count),
                competition=min(tf * 10, 1.0),
                cpc=round(score * 0.1, 2),
                difficulty=difficulty,
                score=round(score, 2),
            )
            keywords.append(kw)

        # 按评分排序，取top_n
        keywords.sort(key=lambda x: x.score, reverse=True)
        return keywords[:top_n]

    def find_related_keywords(self, keyword: str, corpus: List[str],
                               top_n: int = 10) -> List[str]: 17173.baike.tongsou.com 
        """
        从语料库中查找与给定关键词相关的词
        基于共现频率
        """
        related_scores = defaultdict(float)
        keyword_tokens = set(self.preprocessor.tokenize(keyword))

        for text in corpus: 17174.baike.tongsou.com 
            tokens = self.preprocessor.tokenize(text)
            token_set = set(tokens)
            # 如果文本包含关键词，则统计其他词的共现
            if keyword_tokens & token_set:
                for token in tokens:
                    if token not in keyword_tokens:
                        related_scores[token] += 1

        # 排序取top_n
        sorted_related = sorted(related_scores.items(), key=lambda x: x[1], reverse=True)
        return [term for term, _ in sorted_related[:top_n]]

    def _estimate_difficulty(self, count: int, total: int) -> KeywordDifficulty:
        """估算关键词难度"""
        ratio = count / total if total > 0 else 0
        if ratio < 0.001: 17175.baike.tongsou.com 
            return KeywordDifficulty.EASY
        elif ratio < 0.005: 17178.baike.tongsou.com 
            return KeywordDifficulty.MEDIUM
        elif ratio < 0.02: 17177.baike.tongsou.com 
            return KeywordDifficulty.HARD
        else:
            return KeywordDifficulty.VERY_HARD

    def _estimate_search_volume(self, count: int) -> int:
        """粗略估算搜索量"""
        # 简化模型：词频越高，搜索量越大
        return max(100, count * 50)

    def add_keyword(self, term: str, search_volume: int = 0,
                    competition: float = 0.5, cpc: float = 0.0,
                    difficulty: KeywordDifficulty = KeywordDifficulty.MEDIUM,
                    intent: str = "") -> Keyword:
        """手动添加关键词"""
        kw_id = hashlib.md5(term.encode()).hexdigest()[:12]
        kw = Keyword(
            keyword_id=kw_id,
            text=term,
            search_volume=search_volume,
            competition=competition,
            cpc=cpc,
            difficulty=difficulty,
            intent=intent,
            score=0.0,
        )
        self.keywords[kw_id] = kw
        return kw

    def get_keyword(self, term: str) -> Optional[Keyword]:  17179.baike.tongsou.com 
        """获取关键词"""
        kw_id = hashlib.md5(term.encode()).hexdigest()[:12]
        return self.keywords.get(kw_id)


# ==================== 模块4: TF-IDF分析模块 ====================

class TFIDFAnalyzer:
    """
    TF-IDF分析器
    计算词频-逆文档频率，识别重要术语
    """

    def __init__(self, preprocessor: TextPreprocessor):
        self.preprocessor = preprocessor
        self.document_freq: Dict[str, int] = defaultdict(int)
        self.total_documents: int = 0

    def add_document(self, text: str):
        """添加文档到语料库"""
        tokens = set(self.preprocessor.tokenize(text))
        for token in tokens:
            self.document_freq[token] += 1
        self.total_documents += 1

    def compute_tfidf(self, text: str) -> List[TFIDFResult]:
        """
        计算文本的TF-IDF
        """
        tokens = self.preprocessor.tokenize(text)
        if not tokens: 17180.baike.tongsou.com 
            return []

        total_tokens = len(tokens)
        freq = Counter(tokens)

        results = []
        for term, count in freq.items(): 17183.baike.tongsou.com 
            tf = count / total_tokens
            df = self.document_freq.get(term, 0)
            # IDF = log(N / (1 + df))
            idf = math.log(self.total_documents / (1 + df)) if self.total_documents > 0 else 0
            tfidf = tf * idf
            density = count / total_tokens * 100

            results.append(TFIDFResult(
                term=term,
                tf=round(tf, 6),
                idf=round(idf, 6),
                tfidf=round(tfidf, 6),
                count=count,
                density=round(density, 2),
            ))

        # 按TF-IDF降序排列
        results.sort(key=lambda x: x.tfidf, reverse=True)
        return results

    def get_top_terms(self, text: str, top_n: int = 15) -> List[TFIDFResult]:
        """获取Top术语"""
        results = self.compute_tfidf(text)
        return results[:top_n]

    def compare_term_importance(self, text_a: str, text_b: str) -> Dict[str, Tuple[float, float]]:
        """
        比较两段文本中术语的重要性差异
        返回: {term: (tfidf_a, tfidf_b)}
        """
        tfidf_a = {r.term: r.tfidf for r in self.compute_tfidf(text_a)}
        tfidf_b = {r.term: r.tfidf for r in self.compute_tfidf(text_b)}

        all_terms = set(tfidf_a.keys()) | set(tfidf_b.keys())
        comparison = {}
        for term in all_terms: 17184.baike.tongsou.com 
            comparison[term] = (tfidf_a.get(term, 0), tfidf_b.get(term, 0))

        return comparison


# ==================== 模块5: 语义相似度计算 ====================

class SemanticSimilarityCalculator: 17186.baike.tongsou.com 
    """
    语义相似度计算器
    使用简化版词向量方法计算文本相似度
    """

    def __init__(self, preprocessor: TextPreprocessor):
        self.preprocessor = preprocessor

    def cosine_similarity(self, text_a: str, text_b: str) -> float:
        """
        计算余弦相似度
        基于词频向量
        """
        tokens_a = self.preprocessor.tokenize(text_a)
        tokens_b = self.preprocessor.tokenize(text_b)

        if not tokens_a or not tokens_b: 17187.baike.tongsou.com 
            return 0.0

        # 构建词频向量
        freq_a = Counter(tokens_a)
        freq_b = Counter(tokens_b)

        all_terms = set(freq_a.keys()) | set(freq_b.keys())

        # 计算点积
        dot_product = sum(freq_a.get(t, 0) * freq_b.get(t, 0) for t in all_terms)

        # 计算模长
        magnitude_a = math.sqrt(sum(v ** 2 for v in freq_a.values()))
        magnitude_b = math.sqrt(sum(v ** 2 for v in freq_b.values()))

        if magnitude_a == 0 or magnitude_b == 0:
            return 0.0

        return dot_product / (magnitude_a * magnitude_b)

    def jaccard_similarity(self, text_a: str, text_b: str) -> float:
        """
        计算Jaccard相似度
        基于词汇集合
        """
        set_a = set(self.preprocessor.tokenize(text_a))
        set_b = set(self.preprocessor.tokenize(text_b))

        if not set_a or not set_b: 17187.baike.tongsou.com 
            return 0.0

        intersection = set_a & set_b
        union = set_a | set_b

        return len(intersection) / len(union)

    def compute_similarity(self, text_a: str, text_b: str) -> SemanticSimilarity:
        """
        综合计算语义相似度
        """
        cosine = self.cosine_similarity(text_a, text_b)
        jaccard = self.jaccard_similarity(text_a, text_b)

        # 综合评分（加权平均）
        similarity = 0.7 * cosine + 0.3 * jaccard

        # 找出共同术语和缺失术语
        set_a = set(self.preprocessor.tokenize(text_a))
        set_b = set(self.preprocessor.tokenize(text_b))
        common = list(set_a & set_b)
        missing_in_b = list(set_a - set_b)

        return SemanticSimilarity(
            source_text=text_a[:100] + "..." if len(text_a) > 100 else text_a,
            target_text=text_b[:100] + "..." if len(text_b) > 100 else text_b,
            similarity_score=round(similarity, 4),
            common_terms=common[:20],
            missing_terms=missing_in_b[:20],
        )


# ==================== 模块6: 内容评分器 ====================

class ContentScorer: 17189.baike.tongsou.com 
    """
    内容评分器
    多维度评估内容质量
    """

    def __init__(self, preprocessor: TextPreprocessor):
        self.preprocessor = preprocessor
        # 理想参数
        self.ideal_word_count = 1500
        self.ideal_sentence_length = 20
        self.ideal_heading_ratio = 0.1  # 标题数/词数

    def score_content(self, text: str, title: str = "",
                      keywords: List[str] = None) -> ContentScore:
        """
        综合评分内容
        """
        content_id = hashlib.md5(text.encode()).hexdigest()[:12]

        # 各维度评分
        readability = self._score_readability(text)
        keyword_coverage = self._score_keyword_coverage(text, keywords or [])
        structure = self._score_structure(text, title)
        length = self._score_length(text)
        freshness = self._score_freshness(text)

        # 综合评分（加权平均）
        overall = (
            readability * 0.25 +
            keyword_coverage * 0.25 +
            structure * 0.2