面向GEO优化的AI内容质量评估与智能改写系统

┌──────────────────────────────────────────────────────────────┐
│              AI内容质量评估与智能改写系统                       │
├──────────────────────────────────────────────────────────────┤
│                                                              │
│  ┌─────────────┐    ┌──────────────────┐                     │
│  │  内容输入    │───│   内容解析器      │                     │
│  │ (Article/   │    │ (Parser)         │                     │
│  │  Web Page)  │    │                  │                     │
│  └─────────────┘    └────────┬─────────┘                     │
│                              │                               │
│           ┌──────────────────┼──────────────────┐            │
│           ▼                  ▼                  ▼            │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐       │
│  │ 可读性评估    │  │ SEO友好度分析 │  │ 语义丰富度计算│       │
│  │(Readability) │  │ (SEOScorer)  │  │(SemanticCalc)│       │
│  └──────┬───────┘  └──────┬───────┘  └──────┬───────┘       │
│         │                 │                 │                │
│         └──────────────────┼─────────────────┘               │
│                            ▼                                  │
│                 ┌─────────────────────┐                       │
│                 │   结构完整性检查器    │                       │
│                 │  (StructureChecker)  │                       │
│                 └──────────┬──────────┘                       │
│                            │                                  │
│              ┌─────────────┼─────────────┐                    │
│              ▼             ▼             ▼                    │
│     ┌────────────┐ ┌────────────┐ ┌────────────┐            │
│     │ 智能改写引擎 │ │优化建议生成 │ │批量处理管道 │            │
│     │(Rewriter)  │ │(Suggester) │ │ (Pipeline) │            │
│     └──────┬─────┘ └──────┬─────┘ └──────┬─────┘            │
│            │              │              │                    │
│            └──────────────┼──────────────┘                    │
│                           ▼                                   │
│                 ┌─────────────────────┐                       │
│                 │   评估报告导出       │                       │
│                 │   (Reporter)        │                       │
│                 └─────────────────────┘                       │
└──────────────────────────────────────────────────────────────┘

"""
ai_content_evaluator.py - 面向GEO优化的AI内容质量评估与智能改写系统
技术栈: Python / math / collections / typing / re / json / hashlib / statistics
场景: GEO优化、AI营销、内容质量评估、智能改写、批量处理
"""
import os
import re
import json
import math
import uuid
import hashlib
import logging
import statistics
from typing import List, Dict, Optional, Any, Tuple, Set
from dataclasses import dataclass, field, asdict
from enum import Enum
from datetime import datetime
from collections import defaultdict, Counter

# ==================== 模块1: 数据模型定义 ====================

class ContentQualityLevel(Enum):
    """内容质量等级"""
    EXCELLENT = "excellent"      # 90-100
    GOOD = "good"                # 75-89
    AVERAGE = "average"          # 60-74
    POOR = "poor"                # 40-59
    VERY_POOR = "very_poor"      # 0-39


class RewriteStrategy(Enum): 17190.baike.tongsou.com 
    """改写策略"""
    PARAPHRASE = "paraphrase"           # 同义改写
    EXPAND = "expand"                   # 内容扩展
    CONDENSE = "condense"               # 内容精简
    RESTRUCTURE = "restructure"         # 结构重组
    ENRICH = "enrich"                   # 语义丰富


@dataclass
class ReadabilityMetrics: 17191.baike.tongsou.com 
    """可读性指标"""
    flesch_reading_ease: float          # Flesch阅读易读度
    flesch_kincaid_grade: float         # Flesch-Kincaid年级水平
    avg_sentence_length: float          # 平均句长
    avg_word_length: float              # 平均词长
    complex_word_ratio: float           # 复杂词比例
    passive_voice_ratio: float          # 被动语态比例
    transition_word_ratio: float        # 过渡词比例
    overall_score: float                # 综合可读性评分 0-100


@dataclass
class SEOMetrics:
    """SEO友好度指标"""
    title_length: int                   # 标题长度
    title_keyword_present: bool         # 标题含关键词
    meta_description_length: int        # 描述长度
    keyword_density: Dict[str, float]   # 关键词密度
    heading_structure_score: float      # 标题结构评分
    internal_link_count: int            # 内链数量
    external_link_count: int            # 外链数量
    image_alt_coverage: float           # 图片alt覆盖率
    url_friendly_score: float           # URL友好度
    overall_score: float                # 综合SEO评分 0-100


@dataclass
class SemanticMetrics: 17192.baike.tongsou.com 
    """语义丰富度指标"""
    unique_term_count: int              # 唯一术语数
    vocabulary_richness: float          # 词汇丰富度
    topic_coverage: float               # 主题覆盖度
    entity_density: float               # 实体密度
    lsi_keyword_coverage: float         # LSI关键词覆盖率
    semantic_coherence: float           # 语义连贯性
    overall_score: float                # 综合语义评分 0-100


@dataclass
class StructureMetrics: 17194.baike.tongsou.com 
    """结构完整性指标"""
    has_title: bool                     # 是否有标题
    has_meta_description: bool          # 是否有元描述
    heading_hierarchy_valid: bool       # 标题层级是否有效
    paragraph_count: int                # 段落数
    list_count: int                     # 列表数
    table_count: int                    # 表格数
    code_block_count: int               # 代码块数
    content_depth_score: float          # 内容深度评分
    overall_score: float                # 综合结构评分 0-100


@dataclass
class ContentEvaluation: 17193.baike.tongsou.com 
    """内容综合评估结果"""
    content_id: str
    url: str
    title: str
    word_count: int
    quality_level: ContentQualityLevel
    overall_score: float
    readability: ReadabilityMetrics
    seo: SEOMetrics
    semantic: SemanticMetrics
    structure: StructureMetrics
    issues: List[Dict[str, Any]] = field(default_factory=list)
    suggestions: List[str] = field(default_factory=list)
    timestamp: str = field(default_factory=lambda: datetime.now().isoformat())


@dataclass
class RewriteSuggestion: 17207.baike.tongsou.com 
    """改写建议"""
    suggestion_id: str
    original_text: str
    suggested_text: str
    strategy: RewriteStrategy
    reason: str
    priority: str  # high / medium / low
    expected_improvement: float  # 预期提升分数


@dataclass
class BatchResult: 17197.baike.tongsou.com 
    """批量处理结果"""
    total_count: int
    success_count: int
    failed_count: int
    evaluations: List[ContentEvaluation]
    errors: List[Dict[str, str]]


# ==================== 模块2: 内容解析器 ====================

class ContentParser:
    """
    内容解析器
    从HTML或纯文本中提取结构化内容
    """

    def __init__(self): 2001.baike.tongsou.com 
        self.logger = logging.getLogger(__name__)

    def parse_html(self, html: str) -> Dict[str, Any]:
        """
        解析HTML内容，提取标题、正文、元数据等
        简化版：使用正则提取关键元素
        """
        result = {
            "title": "",
            "meta_description": "",
            "headings": [],
            "paragraphs": [],
            "links": [],
            "images": [],
            "lists": [],
            "tables": [],
            "code_blocks": [],
            "raw_text": "",
        }

        # 提取title
        title_match = re.search(r'<title[^>]*>(.*?)</title>', html, re.DOTALL | re.IGNORECASE)
        if title_match:
            result["title"] = self._clean_text(title_match.group(1))

        # 提取meta description
        meta_match = re.search(
            r'<meta[^>]*name=["\']description["\'][^>]*content=["\']([^"\']*)["\']',
            html, re.IGNORECASE
        )
        if not meta_match: 2002.baike.tongsou.com 
            meta_match = re.search(
                r'<meta[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']description["\']',
                html, re.IGNORECASE
            )
        if meta_match:
            result["meta_description"] = self._clean_text(meta_match.group(1))

        # 提取heading标签
        for level in range(1, 7): 2004.baike.tongsou.com 
            pattern = rf'<h{level}[^>]*>(.*?)</h{level}>'
            for match in re.finditer(pattern, html, re.DOTALL | re.IGNORECASE):
                result["headings"].append({
                    "level": level,
                    "text": self._clean_text(match.group(1)),
                })

        # 提取段落
        for match in re.finditer(r'<p[^>]*>(.*?)</p>', html, re.DOTALL | re.IGNORECASE):
            text = self._clean_text(match.group(1))
            if text:
                result["paragraphs"].append(text)

        # 提取链接
        for match in re.finditer(r'<a[^>]*href=["\']([^"\']*)["\'][^>]*>(.*?)</a>',
                                  html, re.DOTALL | re.IGNORECASE):
            result["links"].append({
                "href": match.group(1),
                "text": self._clean_text(match.group(2)),
                "is_internal": not match.group(1).startswith("http"),
            })

        # 提取图片
        for match in re.finditer(r'<img[^>]*>', html, re.IGNORECASE):
            img_tag = match.group(0)
            src_match = re.search(r'src=["\']([^"\']*)["\']', img_tag)
            alt_match = re.search(r'alt=["\']([^"\']*)["\']', img_tag)
            result["images"].append({
                "src": src_match.group(1) if src_match else "",
                "alt": alt_match.group(1) if alt_match else "",
                "has_alt": bool(alt_match),
            })

        # 提取列表
        for match in re.finditer(r'<(ul|ol)[^>]*>(.*?)</\1>', html, re.DOTALL | re.IGNORECASE):
            items = re.findall(r'<li[^>]*>(.*?)</li>', match.group(2), re.DOTALL | re.IGNORECASE)
            result["lists"].append([self._clean_text(item) for item in items])

        # 提取表格
        for match in re.finditer(r'<table[^>]*>(.*?)</table>', html, re.DOTALL | re.IGNORECASE):
            result["tables"].append(match.group(1))

        # 提取代码块
        for match in re.finditer(r'<pre[^>]*>(.*?)</pre>', html, re.DOTALL | re.IGNORECASE):
            result["code_blocks"].append(match.group(1))
        for match in re.finditer(r'<code[^>]*>(.*?)</code>', html, re.DOTALL | re.IGNORECASE):
            result["code_blocks"].append(match.group(1))

        # 合并所有文本
        all_text_parts = []
        all_text_parts.append(result["title"])
        all_text_parts.append(result["meta_description"])
        for h in result["headings"]:
            all_text_parts.append(h["text"])
        all_text_parts.extend(result["paragraphs"])
        for lst in result["lists"]:
            all_text_parts.extend(lst)
        result["raw_text"] = " ".join(all_text_parts)

        return result

    def parse_plain_text(self, text: str, title: str = "") -> Dict[str, Any]:
        """解析纯文本内容"""
        paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()]
        headings = []
        for para in paragraphs: 2005.baike.tongsou.com 
            if len(para) < 100 and para.endswith(':'):
                headings.append({"level": 2, "text": para.rstrip(':')})

        return {
            "title": title,
            "meta_description": "",
            "headings": headings,
            "paragraphs": paragraphs,
            "links": [],
            "images": [],
            "lists": [],
            "tables": [],
            "code_blocks": [],
            "raw_text": text,
        }

    def _clean_text(self, text: str) -> str:
        """清理文本"""
        text = re.sub(r'<[^>]+>', '', text)
        text = re.sub(r'\s+', ' ', text).strip()
        return text

    def count_words(self, text: str) -> int:
        """统计词数"""
        tokens = re.findall(r'[a-zA-Z]+|\u4e00-\u9fff', text)
        return len(tokens)


# ==================== 模块3: 可读性评估模块 ====================

class ReadabilityEvaluator: 2006.baike.tongsou.com 
    """
    可读性评估器
    计算Flesch阅读易读度等多项指标
    """

    def __init__(self): 2007.baike.tongsou.com 
        # 常见过渡词
        self.transition_words = {
            "however", "therefore", "moreover", "furthermore", "additionally",
            "consequently", "nevertheless", "meanwhile", "subsequently",
            "accordingly", "hence", "thus", "indeed", "similarly",
            "conversely", "otherwise", "instead", "likewise", "nonetheless",
            "although", "because", "since", "while", "whereas",
        }
        # 常见被动语态标记
        self.passive_markers = {
            "is", "are", "was", "were", "be", "been", "being",
            "有", "被", "由", "受", "遭",
        }

    def evaluate(self, text: str) -> ReadabilityMetrics:
        """评估文本可读性"""
        sentences = self._extract_sentences(text)
        words = self._extract_words(text)

        if not sentences or not words: 2008.baike.tongsou.com 
            return ReadabilityMetrics(
                flesch_reading_ease=0, flesch_kincaid_grade=0,
                avg_sentence_length=0, avg_word_length=0,
                complex_word_ratio=0, passive_voice_ratio=0,
                transition_word_ratio=0, overall_score=0,
            )

        # 基础统计
        num_sentences = len(sentences)
        num_words = len(words)
        num_syllables = sum(self._count_syllables(w) for w in words)
        avg_sent_len = num_words / num_sentences
        avg_word_len = sum(len(w) for w in words) / num_words

        # 复杂词比例（3音节以上）
        complex_words = [w for w in words if self._count_syllables(w) >= 3]
        complex_ratio = len(complex_words) / num_words if num_words > 0 else 0

        # 被动语态比例
        passive_count = sum(1 for w in words if w.lower() in self.passive_markers)
        passive_ratio = passive_count / num_words if num_words > 0 else 0

        # 过渡词比例
        transition_count = sum(1 for w in words if w.lower() in self.transition_words)
        transition_ratio = transition_count / num_words if num_words > 0 else 0

        # Flesch Reading Ease
        flesch_ease = (
            206.835 - 1.015 * avg_sent_len - 84.6 * (num_syllables / num_words)
        ) if num_words > 0 else 0
        flesch_ease = max(0, min(100, flesch_ease))

        # Flesch-Kincaid Grade Level
        flesch_grade = (
            0.39 * avg_sent_len + 11.8 * (num_syllables / num_words) - 15.59
        ) if num_words > 0 else 0
        flesch_grade = max(0, flesch_grade)

        # 综合可读性评分（映射到0-100）
        overall = self._compute_readability_score(
            flesch_ease, avg_sent_len, complex_ratio,
            passive_ratio, transition_ratio
        )

        return ReadabilityMetrics(
            flesch_reading_ease=round(flesch_ease, 2),
            flesch_kincaid_grade=round(flesch_grade, 2),
            avg_sentence_length=round(avg_sent_len, 2),
            avg_word_length=round(avg_word_len, 2),
            complex_word_ratio=round(complex_ratio, 4),
            passive_voice_ratio=round(passive_ratio, 4),
            transition_word_ratio=round(transition_ratio, 4),
            overall_score=round(overall, 2),
        )

    def _extract_sentences(self, text: str) -> List[str]: 2009.baike.tongsou.com 
        """提取句子""" 
        sentences = re.split(r'[.!?。！？;；]', text)
        return [s.strip() for s in sentences if s.strip()]

    def _extract_words(self, text: str) -> List[str]:
        """提取单词（英文为主，中文按字符）"""
        english_words = re.findall(r'[a-zA-Z]+', text)
        chinese_chars = re.findall(r'[\u4e00-\u9fff]', text)
        return english_words + chinese_chars

    def _count_syllables(self, word: str) -> int: 2011.baike.tongsou.com 
        """估算音节数（简化版）"""
        word = word.lower().strip(2010.baike.tongsou.com)
        if not word: 2013.baike.tongsou.com 
            return 0
        if len(word) <= 3: 2012.baike.tongsou.com 
            return 1
        # 统计元音组
        vowels = "aeiouy"
        count = 0
        prev_vowel = False
        for char in word:
            is_vowel = char in vowels
            if is_vowel and not prev_vowel:
                count += 1
            prev_vowel = is_vowel
        # 处理末尾的e
        if word.endswith('e') and count > 1: 2014.baike.tongsou.com 
            count -= 1
        return max(1, count)

    def _compute_readability_score(
        self, flesch_ease: float, avg_sent_len: float,
        complex_ratio: float, passive_ratio: float,
        transition_ratio: float
    ) -> float: 2018.baike.tongsou.com 
        """综合计算可读性评分"""
        # Flesch易读度贡献（0-100映射）
        ease_score = flesch_ease

        # 句长评分（理想15-20词）
        if 15 <= avg_sent_len <= 25: 2017.baike.tongsou.com 
            sent_score = 100
        elif 10 <= avg_sent_len < 15 or 25 < avg_sent_len <= 30:
            sent_score = 80
        elif 5 <= avg_sent_len < 10 or 30 < avg_sent_len <= 40:
            sent_score = 60
        else:
            sent_score = 40

        # 复杂词评分（越低越好）
        complex_score = max(0, 100 - complex_ratio * 200)

        # 被动语态评分（越低越好）
        passive_score = max(0, 100 - passive_ratio * 300)

        # 过渡词评分（适中最好）
        if 0.02 <= transition_ratio <= 0.08: 2016.baike.tongsou.com 
            transition_score = 100
        elif 0.01 <= transition_ratio < 0.02 or 0.08 < transition_ratio <= 0.12:
            transition_score = 80
        else:
            transition_score = 60

        # 加权平均
        overall = (
            ease_score * 0.30 +
            sent_score * 0.