58_大模型评估与评测：构建科学的多维度评测体系

# 混合评估方法示例代码
import pandas as pd
import numpy as np
from sklearn.metrics import precision_recall_fscore_support
from scipy import stats

def calculate_weighted_score(evaluations, weights=None):
    """
    计算多维度评估的加权总分
    
    参数:
        evaluations: 包含各维度评估结果的字典
        weights: 各维度的权重字典
    
    返回:
        加权总分和各维度得分详情
    """
    # 如果没有提供权重，使用均等权重
    if weights is None:
        weights = {dim: 1.0/len(evaluations) for dim in evaluations.keys()}
    
    # 验证权重是否有效
    total_weight = sum(weights.values())
    if not np.isclose(total_weight, 1.0):
        print(f"Warning: Weights do not sum to 1.0, normalizing. Total weight: {total_weight}")
        # 归一化权重
        weights = {dim: w/total_weight for dim, w in weights.items()}
    
    # 计算加权总分
    weighted_score = 0
    details = {}
    
    for dimension, score in evaluations.items():
        weight = weights.get(dimension, 0)
        weighted_contribution = score * weight
        weighted_score += weighted_contribution
        
        details[dimension] = {
            'score': score,
            'weight': weight,
            'contribution': weighted_contribution
        }
    
    return {
        'total_score': weighted_score,
        'details': details,
        'weights': weights
    }

def normalize_scores(scores, min_val=0, max_val=100):
    """
    对不同来源的分数进行归一化
    
    参数:
        scores: 包含原始分数的字典
        min_val: 归一化后的最小值
        max_val: 归一化后的最大值
    
    返回:
        归一化后的分数字典
    """
    # 找出所有分数的最小和最大值
    all_scores = list(scores.values())
    original_min = min(all_scores)
    original_max = max(all_scores)
    
    # 避免除以零
    if original_min == original_max:
        return {k: (min_val + max_val) / 2 for k in scores.keys()}
    
    # 执行线性归一化
    normalized = {}
    for key, score in scores.items():
        normalized_score = ((score - original_min) / (original_max - original_min)) * (max_val - min_val) + min_val
        normalized[key] = normalized_score
    
    return normalized

def calculate_inter_rater_reliability(ratings):
    """
    计算人工评估的评分者间信度 (Cohen's Kappa for 2 raters)
    
    参数:
        ratings: 包含多个评分者对多个项目评分的DataFrame
    
    返回:
        评分者间信度指标
    """
    # 提取两个评分者的评分
    if len(ratings.columns) < 2:
        return {"error": "至少需要两个评分者的评分"}
    
    rater1 = ratings.iloc[:, 0]
    rater2 = ratings.iloc[:, 1]
    
    # 计算Cohen's Kappa
    # 这里使用简化实现，实际应用中可使用scikit-learn的实现
    from sklearn.metrics import cohen_kappa_score
    kappa = cohen_kappa_score(rater1, rater2)
    
    # 如果有更多评分者，计算Fleiss' Kappa
    if len(ratings.columns) > 2:
        from statsmodels.stats.inter_rater import fleiss_kappa
        # 转换为适合Fleiss' Kappa的格式
        unique_ratings = sorted(list(set(ratings.values.flatten())))
        rating_to_idx = {r: i for i, r in enumerate(unique_ratings)}
        
        # 创建n x k矩阵，n是项目数，k是评分类别数
        n_items = len(ratings)
        n_categories = len(unique_ratings)
        kappa_matrix = np.zeros((n_items, n_categories))
        
        for i in range(n_items):
            item_ratings = ratings.iloc[i].values
            for r in item_ratings:
                kappa_matrix[i, rating_to_idx[r]] += 1
        
        fleiss = fleiss_kappa(kappa_matrix)
        
        return {
            "cohens_kappa": kappa,  # 前两个评分者的Cohen's Kappa
            "fleiss_kappa": fleiss,  # 所有评分者的Fleiss' Kappa
            "n_raters": len(ratings.columns),
            "n_items": n_items
        }
    
    return {
        "cohens_kappa": kappa,
        "n_raters": 2,
        "n_items": len(ratings)
    }

def mixed_evaluation_pipeline(model_responses, reference_answers, human_ratings=None):
    """
    混合评估流水线，结合自动化评估和人工评估
    
    参数:
        model_responses: 模型的回复列表
        reference_answers: 参考答案列表
        human_ratings: 人工评分DataFrame（可选）
    
    返回:
        综合评估结果
    """
    results = {}
    
    # 1. 自动化评估指标
    # 这里只是示例，实际应用中需要根据具体任务设计指标
    automated_scores = {}
    
    # 示例：计算精确率、召回率和F1分数
    # 注意：这需要将文本转换为可比较的格式，这里简化处理
    y_true = [1 if "正确" in ans else 0 for ans in reference_answers]
    y_pred = [1 if "正确" in resp else 0 for resp in model_responses]
    
    precision, recall, f1, _ = precision_recall_fscore_support(
        y_true, y_pred, average='binary', zero_division=0
    )
    
    automated_scores["precision"] = precision * 100
    automated_scores["recall"] = recall * 100
    automated_scores["f1_score"] = f1 * 100
    
    # 计算响应长度统计
    response_lengths = [len(resp.split()) for resp in model_responses]
    automated_scores["avg_response_length"] = np.mean(response_lengths)
    automated_scores["std_response_length"] = np.std(response_lengths)
    
    results["automated_evaluation"] = automated_scores
    
    # 2. 人工评估（如果提供）
    if human_ratings is not None:
        # 计算人工评分的统计信息
        avg_ratings = human_ratings.mean(axis=1)
        results["human_evaluation"] = {
            "average_score": avg_ratings.mean(),
            "std_score": avg_ratings.std(),
            "min_score": avg_ratings.min(),
            "max_score": avg_ratings.max(),
            "reliability": calculate_inter_rater_reliability(human_ratings)
        }
        
        # 将人工评分也加入到综合评分
        automated_scores["human_score"] = avg_ratings.mean()
    
    # 3. 综合评分
    # 定义各指标的权重
    weights = {
        "precision": 0.2,
        "recall": 0.2,
        "f1_score": 0.3,
    }
    
    # 如果有人工评分，给予较高权重
    if "human_score" in automated_scores:
        weights["human_score"] = 0.5
        # 调整其他权重
        remaining_weight = 0.5
        for key in weights:
            if key != "human_score":
                weights[key] = weights[key] * remaining_weight / 0.7  # 原始非人工评分权重总和为0.7
    
    # 计算加权总分
    results["comprehensive_score"] = calculate_weighted_score(automated_scores, weights)
    
    # 4. 结果汇总
    results["summary"] = {
        "total_items_evaluated": len(model_responses),
        "evaluation_timestamp": pd.Timestamp.now().isoformat(),
        "final_score": results["comprehensive_score"]["total_score"]
    }
    
    return results

# 示例使用
if __name__ == "__main__":
    # 模拟数据
    model_responses = [
        "这是一个正确的回答，包含了所有必要的信息。",
        "部分正确，但缺少一些关键细节。",
        "完全错误的回答，与问题无关。",
        "正确但过于简略的回答。",
        "非常全面且准确的回答。"
    ]
    
    reference_answers = [
        "参考答案1：应该包含所有必要信息。",
        "参考答案2：需要包含关键细节。",
        "参考答案3：正确的回答应与问题相关。",
        "参考答案4：应该详细而准确。",
        "参考答案5：全面准确是最佳标准。"
    ]
    
    # 模拟人工评分（3个评分者对5个回答的评分，1-5分）
    human_ratings = pd.DataFrame({
        "rater1": [4, 3, 1, 3, 5],
        "rater2": [5, 2, 1, 4, 5],
        "rater3": [4, 3, 2, 3, 4]
    })
    
    # 执行混合评估
    evaluation_results = mixed_evaluation_pipeline(model_responses, reference_answers, human_ratings)
    
    # 打印结果
    print("\n评估结果摘要:")
    print(f"评估项目数: {evaluation_results['summary']['total_items_evaluated']}")
    print(f"评估时间: {evaluation_results['summary']['evaluation_timestamp']}")
    print(f"最终综合得分: {evaluation_results['summary']['final_score']:.2f}/100")
    
    print("\n自动化评估结果:")
    for metric, score in evaluation_results['automated_evaluation'].items():
        if metric != "human_score":  # 人工评分单独显示
            print(f"  {metric}: {score:.2f}")
    
    print("\n人工评估结果:")
    human_eval = evaluation_results['human_evaluation']
    print(f"  平均分: {human_eval['average_score']:.2f}/5")
    print(f"  标准差: {human_eval['std_score']:.2f}")
    print(f"  评分者间信度 (Fleiss' Kappa): {human_eval['reliability']['fleiss_kappa']:.3f}")
    
    print("\n综合评分详情:")
    details = evaluation_results['comprehensive_score']['details']
    for dim, info in details.items():
        print(f"  {dim}: 得分={info['score']:.2f}, 权重={info['weight']:.2f}, 贡献={info['contribution']:.2f}")

# 多维度评估指标计算示例
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.metrics import confusion_matrix, classification_report

def calculate_text_quality_metrics(model_outputs, references):
    """
    计算文本生成质量相关指标
    
    参数:
        model_outputs: 模型生成的文本列表
        references: 参考答案列表
    
    返回:
        包含各种质量指标的字典
    """
    metrics = {}
    
    # 1. 简单的准确性评估（这里使用精确匹配作为示例）
    exact_matches = sum(1 for pred, ref in zip(model_outputs, references) if pred.strip() == ref.strip())
    metrics['exact_accuracy'] = exact_matches / len(model_outputs) if model_outputs else 0
    
    # 2. 文本长度统计
    pred_lengths = [len(text.split()) for text in model_outputs]
    ref_lengths = [len(text.split()) for text in references]
    
    metrics['avg_pred_length'] = np.mean(pred_lengths) if pred_lengths else 0
    metrics['avg_ref_length'] = np.mean(ref_lengths) if ref_lengths else 0
    metrics['length_ratio'] = metrics['avg_pred_length'] / metrics['avg_ref_length'] if metrics['avg_ref_length'] > 0 else 0
    
    # 3. 词汇多样性 - Distinct-n指标
    def calculate_distinct_n(texts, n=1):
        """计算Distinct-n指标"""
        if not texts:
            return 0
        
        all_ngrams = set()
        total_ngrams = 0
        
        for text in texts:
            words = text.split()
            if len(words) < n:
                continue
            
            ngrams = [tuple(words[i:i+n]) for i in range(len(words)-n+1)]
            all_ngrams.update(ngrams)
            total_ngrams += len(ngrams)
        
        return len(all_ngrams) / total_ngrams if total_ngrams > 0 else 0
    
    metrics['distinct_1'] = calculate_distinct_n(model_outputs, n=1)
    metrics['distinct_2'] = calculate_distinct_n(model_outputs, n=2)
    metrics['distinct_3'] = calculate_distinct_n(model_outputs, n=3)
    
    # 4. 关键词覆盖率（简化示例）
    # 实际应用中需要更复杂的关键词提取和匹配策略
    keyword_coverage = []
    for pred, ref in zip(model_outputs, references):
        ref_words = set(ref.lower().split())
        pred_words = set(pred.lower().split())
        if ref_words:
            coverage = len(ref_words.intersection(pred_words)) / len(ref_words)
            keyword_coverage.append(coverage)
    
    metrics['avg_keyword_coverage'] = np.mean(keyword_coverage) if keyword_coverage else 0
    
    return metrics

def calculate_performance_metrics(latency_data, throughput_data, memory_usage):
    """
    计算性能相关指标
    
    参数:
        latency_data: 延迟数据列表（秒）
        throughput_data: 吞吐量数据列表（token/秒）
        memory_usage: 内存使用数据列表（GB）
    
    返回:
        包含性能指标的字典
    """
    metrics = {}
    
    # 延迟统计
    if latency_data:
        metrics['avg_latency'] = np.mean(latency_data)
        metrics['p50_latency'] = np.percentile(latency_data, 50)
        metrics['p90_latency'] = np.percentile(latency_data, 90)
        metrics['p99_latency'] = np.percentile(latency_data, 99)
        metrics['max_latency'] = np.max(latency_data)
        metrics['min_latency'] = np.min(latency_data)
    
    # 吞吐量统计
    if throughput_data:
        metrics['avg_throughput'] = np.mean(throughput_data)
        metrics['max_throughput'] = np.max(throughput_data)
        metrics['min_throughput'] = np.min(throughput_data)
        metrics['std_throughput'] = np.std(throughput_data)
    
    # 内存使用统计
    if memory_usage:
        metrics['avg_memory_usage'] = np.mean(memory_usage)
        metrics['peak_memory_usage'] = np.max(memory_usage)
        metrics['memory_utilization'] = np.mean(memory_usage) / 32.0  # 假设GPU内存为32GB
    
    # 计算效率指标
    if latency_data and memory_usage:
        metrics['latency_per_gb'] = np.mean(latency_data) / np.mean(memory_usage)
    
    if throughput_data and memory_usage:
        metrics['throughput_per_gb'] = np.mean(throughput_data) / np.mean(memory_usage)
    
    return metrics

def calculate_user_experience_metrics(user_feedback):
    """
    计算用户体验相关指标
    
    参数:
        user_feedback: 包含用户反馈的DataFrame，应包含'satisfaction'（1-5分）、'completion'（是否完成任务）等列
    
    返回:
        包含用户体验指标的字典
    """
    metrics = {}
    
    # 用户满意度指标
    if 'satisfaction' in user_feedback.columns:
        metrics['avg_satisfaction'] = user_feedback['satisfaction'].mean()
        metrics['satisfaction_distribution'] = user_feedback['satisfaction'].value_counts().to_dict()
        metrics['satisfaction_90+_percentage'] = (user_feedback['satisfaction'] >= 4.5).mean() * 100
    
    # 任务完成指标
    if 'completion' in user_feedback.columns:
        metrics['task_completion_rate'] = user_feedback['completion'].mean() * 100
    
    # 使用频率指标
    if 'session_count' in user_feedback.columns:
        metrics['avg_sessions_per_user'] = user_feedback['session_count'].mean()
        metrics['active_users'] = (user_feedback['session_count'] > 0).sum()
    
    # 净推荐值 (NPS) 计算
    if 'recommendation_willingness' in user_feedback.columns:  # 0-10分
        promoters = (user_feedback['recommendation_willingness'] >= 9).sum()
        detractors = (user_feedback['recommendation_willingness'] <= 6).sum()
        total_respondents = len(user_feedback)
        
        if total_respondents > 0:
            metrics['nps_score'] = ((promoters - detractors) / total_respondents) * 100
    
    # 功能使用分布
    if 'features_used' in user_feedback.columns:
        # 假设features_used是包含使用功能列表的列
        feature_counts = {}
        for features in user_feedback['features_used']:
            if isinstance(features, list):
                for feature in features:
                    feature_counts[feature] = feature_counts.get(feature, 0) + 1
        
        metrics['feature_usage_distribution'] = feature_counts
        metrics['avg_features_per_session'] = np.mean([len(f) for f in user_feedback['features_used'] if isinstance(f, list)])
    
    return metrics

def normalize_and_combine_metrics(metrics_dict, weights=None):
    """
    归一化并组合多个维度的指标
    
    参数:
        metrics_dict: 包含各维度指标的字典，格式为 {"维度": {"指标名": 指标值}}
        weights: 各维度的权重字典
    
    返回:
        归一化后的综合指标
    """
    if weights is None:
        # 默认均等权重
        weights = {dim: 1.0 / len(metrics_dict) for dim in metrics_dict.keys()}
    
    # 确保权重和为1
    total_weight = sum(weights.values())
    if not np.isclose(total_weight, 1.0):
        weights = {dim: w / total_weight for dim, w in weights.items()}
    
    # 存储归一化后的指标
    normalized_metrics = {}
    dimension_scores = {}
    
    # 对每个维度的指标进行归一化
    for dimension, metrics in metrics_dict.items():
        # 转换为DataFrame以便处理
        df = pd.DataFrame([metrics])
        
        # 选择数值列进行归一化
        numeric_cols = df.select_dtypes(include=[np.number]).columns
        
        if len(numeric_cols) > 0:
            # 使用MinMaxScaler将所有指标归一化到[0, 1]
            scaler = MinMaxScaler()
            scaled_values = scaler.fit_transform(df[numeric_cols])
            
            # 创建归一化后的指标字典
            norm_metrics = {col: scaled_values[0, i] for i, col in enumerate(numeric_cols)}
            normalized_metrics[dimension] = norm_metrics
            
            # 计算该维度的平均分
            dimension_scores[dimension] = np.mean(list(norm_metrics.values()))
        else:
            normalized_metrics[dimension] = {}
            dimension_scores[dimension] = 0
    
    # 计算加权综合得分
    weighted_score = sum(dimension_scores[dim] * weights[dim] for dim in dimension_scores)
    
    return {
        "dimension_scores": dimension_scores,
        "normalized_metrics": normalized_metrics,
        "weights": weights,
        "overall_score": weighted_score
    }

def visualize_evaluation_results(combined_metrics, output_file=None):
    """
    可视化评估结果
    
    参数:
        combined_metrics: 由normalize_and_combine_metrics函数返回的结果
        output_file: 输出文件路径，如果为None则显示图表
    """
    # 设置可视化风格
    plt.style.use('seaborn-v0_8-whitegrid')
    
    # 创建一个包含多个子图的图表
    fig = plt.figure(figsize=(20, 15))
    
    # 1. 雷达图展示各维度得分
    ax1 = fig.add_subplot(2, 2, 1, polar=True)
    
    dimensions = list(combined_metrics['dimension_scores'].keys())
    scores = list(combined_metrics['dimension_scores'].values())
    
    # 闭合雷达图
    angles = np.linspace(0, 2*np.pi, len(dimensions), endpoint=False).tolist()
    scores = scores + [scores[0]]
    angles = angles + [angles[0]]
    dimensions = dimensions + [dimensions[0]]
    
    ax1.plot(angles, scores, 'o-', linewidth=2)
    ax1.fill(angles, scores, alpha=0.25)
    ax1.set_thetagrids(np.degrees(angles[:-1]), dimensions[:-1])
    ax1.set_ylim(0, 1)
    ax1.set_title('各维度得分雷达图', fontsize=15, pad=20)
    
    # 2. 柱状图展示各维度得分
    ax2 = fig.add_subplot(2, 2, 2)
    ax2.bar(dimensions[:-1], [s * 100 for s in combined_metrics['dimension_scores'].values()])
    ax2.set_ylim(0, 100)
    ax2.set_ylabel('得分 (%)')
    ax2.set_title('各维度得分柱状图', fontsize=15)
    
    # 添加数值标签
    for i, v in enumerate([s * 100 for s in combined_metrics['dimension_scores'].values()]):
        ax2.text(i, v + 1, f'{v:.1f}', ha='center')
    
    # 3. 饼图展示权重分布
    ax3 = fig.add_subplot(2, 2, 3)
    ax3.pie(combined_metrics['weights'].values(), labels=combined_metrics['weights'].keys(), 
            autopct='%1.1f%%', startangle=90)
    ax3.set_title('各维度权重分布', fontsize=15)
    ax3.axis('equal')  # 确保饼图是圆的
    
    # 4. 总体得分
    ax4 = fig.add_subplot(2, 2, 4)
    ax4.axis('off')  # 不显示坐标轴
    
    overall_score = combined_metrics['overall_score'] * 100
    ax4.text(0.5, 0.5, f'综合得分\n{overall_score:.1f}/100', 
             fontsize=30, ha='center', va='center',
             bbox=dict(boxstyle='round,pad=1', facecolor='#f0f0f0', alpha=0.5))
    
    # 调整布局
    plt.tight_layout()
    
    # 保存或显示图表
    if output_file:
        plt.savefig(output_file, dpi=300, bbox_inches='tight')
        print(f"评估结果可视化已保存至: {output_file}")
    else:
        plt.show()

# 示例使用
if __name__ == "__main__":
    # 模拟数据
    # 1. 文本质量评估数据
    model_outputs = [
        "这是一个高质量的回答，准确且全面。",
        "回答部分正确，但缺少一些关键信息。",
        "完全错误的回答，与问题无关。",
        "回答正确但过于简略。",
        "非常全面且详细的正确回答。"
    ]
    
    references = [
        "这是一个准确且全面的高质量回答。",
        "正确的回答应包含所有关键信息。",
        "回答应与问题相关且准确。",
        "回答应详细且准确。",
        "非常全面且详细的正确回答是最佳的。"
    ]
    
    # 2. 性能评估数据
    import random
    latency_data = [random.uniform(0.5, 2.5) for _ in range(100)]  # 模拟100个延迟数据
    throughput_data = [random.uniform(50, 150) for _ in range(100)]  # 模拟吞吐量
    memory_usage = [random.uniform(10, 25) for _ in range(100)]  # 模拟内存使用
    
    # 3. 用户体验数据
    user_feedback = pd.DataFrame({
        'user_id': range(1, 51),
        'satisfaction': [random.randint(1, 5) for _ in range(50)],
        'completion': [random.choice([True, True, True, False]) for _ in range(50)],
        'session_count': [random.randint(1, 10) for _ in range(50)],
        'recommendation_willingness': [random.randint(0, 10) for _ in range(50)],
        'features_used': [[random.choice(['feature1', 'feature2', 'feature3', 'feature4']) for _ in range(random.randint(1, 4))] for _ in range(50)]
    })
    
    # 计算各维度指标
    text_metrics = calculate_text_quality_metrics(model_outputs, references)
    performance_metrics = calculate_performance_metrics(latency_data, throughput_data, memory_usage)
    ux_metrics = calculate_user_experience_metrics(user_feedback)
    
    # 组合各维度指标
    metrics_dict = {
        "文本质量": text_metrics,
        "技术性能": performance_metrics,
        "用户体验": {k: v for k, v in ux_metrics.items() if isinstance(v, (int, float))}
    }
    
    # 设置维度权重
    weights = {
        "文本质量": 0.4,
        "技术性能": 0.3,
        "用户体验": 0.3
    }
    
    # 归一化并计算综合指标
    combined_metrics = normalize_and_combine_metrics(metrics_dict, weights)
    
    # 打印综合结果
    print("\n综合评估结果:")
    print(f"总体得分: {combined_metrics['overall_score'] * 100:.1f}/100")
    
    print("\n各维度得分:")
    for dim, score in combined_metrics['dimension_scores'].items():
        print(f"  {dim}: {score * 100:.1f}/100")
    
    print("\n各维度权重:")
    for dim, weight in combined_metrics['weights'].items():
        print(f"  {dim}: {weight * 100:.1f}%")
    
    # 可视化结果
    # visualize_evaluation_results(combined_metrics, "evaluation_results.png")
    print("可视化函数已准备就绪，可通过取消注释上述行来生成可视化图表")

# 医疗领域评估基准示例代码
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score

class MedicalLLMEvaluator:
    """
    医疗领域大模型评估器
    """
    def __init__(self, reference_data, ground_truth, domain_experts=None):
        self.reference_data = reference_data  # 参考医疗案例和指南
        self.ground_truth = ground_truth      # 真实诊断和治疗方案
        self.domain_experts = domain_experts  # 领域专家评分（可选）
        self.evaluation_results = {}
    
    def evaluate_diagnostic_accuracy(self, model_diagnoses):
        """
        评估诊断准确性
        """
        # 提取关键诊断信息进行比较
        true_labels = self._extract_diagnostic_labels(self.ground_truth)
        pred_labels = self._extract_diagnostic_labels(model_diagnoses)
        
        # 计算各种评估指标
        accuracy = accuracy_score(true_labels, pred_labels)
        precision, recall, f1, _ = precision_recall_fscore_support(
            true_labels, pred_labels, average='macro', zero_division=0
        )
        
        # 如果是二分类问题，计算AUC
        if len(np.unique(true_labels)) == 2:
            auc = roc_auc_score(true_labels, [1 if p == true_labels[0] else 0 for p in pred_labels])
        else:
            auc = None
        
        results = {
            "accuracy": accuracy,
            "precision": precision,
            "recall": recall,
            "f1_score": f1
        }
        
        if auc is not None:
            results["auc"] = auc
        
        self.evaluation_results["diagnostic_accuracy"] = results
        return results
    
    def evaluate_treatment_recommendations(self, model_recommendations):
        """
        评估治疗建议的质量
        """
        # 提取治疗方案的关键组成部分
        true_treatments = self._extract_treatment_components(self.ground_truth)
        pred_treatments = self._extract_treatment_components(model_recommendations)
        
        # 计算关键药物匹配度
        medication_precision = []
        medication_recall = []
        
        for true_meds, pred_meds in zip(true_treatments["medications"], pred_treatments["medications"]):
            if not true_meds and not pred_meds:
                med_prec, med_rec = 1.0, 1.0
            elif not true_meds or not pred_meds:
                med_prec, med_rec = 0.0, 0.0
            else:
                intersection = set(true_meds) & set(pred_meds)
                med_prec = len(intersection) / len(set(pred_meds))
                med_rec = len(intersection) / len(set(true_meds))
            
            medication_precision.append(med_prec)
            medication_recall.append(med_rec)
        
        # 计算剂量准确性（简化版）
        dosage_accuracy = []
        for true_dosage, pred_dosage in zip(true_treatments["dosages"], pred_treatments["dosages"]):
            # 计算剂量偏差百分比
            if true_dosage > 0:
                deviation = abs(pred_dosage - true_dosage) / true_dosage
                # 如果偏差小于20%，认为准确
                dosage_acc = 1.0 if deviation <= 0.2 else max(0, 1 - deviation)
            else:
                dosage_acc = 1.0 if pred_dosage == 0 else 0.0
            dosage_accuracy.append(dosage_acc)
        
        results = {
            "avg_medication_precision": np.mean(medication_precision),
            "avg_medication_recall": np.mean(medication_recall),
            "avg_medication_f1": 2 * np.mean(medication_precision) * np.mean(medication_recall) / \
                                (np.mean(medication_precision) + np.mean(medication_recall) + 1e-10),
            "avg_dosage_accuracy": np.mean(dosage_accuracy)
        }
        
        self.evaluation_results["treatment_quality"] = results
        return results
    
    def evaluate_medical_knowledge(self, model_responses):
        """
        评估医学知识的准确性
        """
        knowledge_scores = []
        
        for i, response in enumerate(model_responses):
            # 这里简化处理，实际应用中需要更复杂的知识验证
            # 基于参考资料检查关键知识点
            reference = self.reference_data[i]
            
            # 提取关键知识点
            key_points = self._extract_key_medical_points(reference)
            
            # 检查模型回答中包含的关键知识点数量
            covered_points = 0
            total_points = len(key_points)
            
            for point in key_points:
                if self._check_knowledge_point_coverage(response, point):
                    covered_points += 1
            
            # 计算知识点覆盖率
            if total_points > 0:
                coverage_score = covered_points / total_points
            else:
                coverage_score = 1.0
            
            knowledge_scores.append(coverage_score)
        
        avg_knowledge_score = np.mean(knowledge_scores)
        self.evaluation_results["knowledge_accuracy"] = {
            "average_score": avg_knowledge_score,
            "detailed_scores": knowledge_scores
        }
        return avg_knowledge_score
    
    def evaluate_safety_compliance(self, model_responses):
        """
        评估安全合规性
        """
        safety_issues = {
            "misinformation": 0,
            "dangerous_recommendations": 0,
            "ethical_violations": 0,
            "incomplete_disclosures": 0
        }
        
        for response in model_responses:
            # 检查错误信息
            if self._detect_misinformation(response):
                safety_issues["misinformation"] += 1
            
            # 检查危险建议
            if self._detect_dangerous_recommendations(response):
                safety_issues["dangerous_recommendations"] += 1
            
            # 检查伦理违规
            if self._detect_ethical_violations(response):
                safety_issues["ethical_violations"] += 1
            
            # 检查披露是否完整
            if not self._check_complete_disclosures(response):
                safety_issues["incomplete_disclosures"] += 1
        
        # 计算安全合规分数
        total_responses = len(model_responses)
        safety_score = 1.0 - sum(safety_issues.values()) / (total_responses * len(safety_issues))
        
        self.evaluation_results["safety_compliance"] = {
            "safety_score": safety_score,
            "issues": {k: v/total_responses for k, v in safety_issues.items()}
        }
        return safety_score
    
    def get_medical_comprehensive_score(self):
        """
        计算医疗领域综合评分
        """
        weights = {
            "diagnostic_accuracy": 0.35,
            "treatment_quality": 0.30,
            "knowledge_accuracy": 0.20,
            "safety_compliance": 0.15
        }
        
        comprehensive_score = 0.0
        
        # 诊断准确性权重
        if "diagnostic_accuracy" in self.evaluation_results:
            diag_score = self.evaluation_results["diagnostic_accuracy"].get("f1_score", 0)
            comprehensive_score += diag_score * weights["diagnostic_accuracy"]
        
        # 治疗质量权重
        if "treatment_quality" in self.evaluation_results:
            treat_score = self.evaluation_results["treatment_quality"].get("avg_medication_f1", 0)
            comprehensive_score += treat_score * weights["treatment_quality"]
        
        # 知识准确性权重
        if "knowledge_accuracy" in self.evaluation_results:
            knowl_score = self.evaluation_results["knowledge_accuracy"].get("average_score", 0)
            comprehensive_score += knowl_score * weights["knowledge_accuracy"]
        
        # 安全合规权重
        if "safety_compliance" in self.evaluation_results:
            safety_score = self.evaluation_results["safety_compliance"].get("safety_score", 0)
            comprehensive_score += safety_score * weights["safety_compliance"]
        
        return {
            "comprehensive_score": comprehensive_score,
            "component_scores": {
                "diagnostic_accuracy": self.evaluation_results.get("diagnostic_accuracy", {}).get("f1_score", 0),
                "treatment_quality": self.evaluation_results.get("treatment_quality", {}).get("avg_medication_f1", 0),
                "knowledge_accuracy": self.evaluation_results.get("knowledge_accuracy", {}).get("average_score", 0),
                "safety_compliance": self.evaluation_results.get("safety_compliance", {}).get("safety_score", 0)
            },
            "weights": weights
        }
    
    # 辅助方法（简化实现）
    def _extract_diagnostic_labels(self, data):
        return [item.get("diagnosis", "unknown") for item in data]
    
    def _extract_treatment_components(self, data):
        medications = []
        dosages = []
        for item in data:
            meds = item.get("medications", [])
            medication_list = [med.get("name", "") for med in meds if "name" in med]
            medication_dosages = [med.get("dosage", 0) for med in meds if "dosage" in med]
            medications.append(medication_list)
            dosages.append(sum(medication_dosages) if medication_dosages else 0)
        return {"medications": medications, "dosages": dosages}
    
    def _extract_key_medical_points(self, reference):
        return reference.get("key_points", [])
    
    def _check_knowledge_point_coverage(self, response, point):
        return point.lower() in response.lower()
    
    def _detect_misinformation(self, response):
        # 简化实现，实际应用中需要更复杂的检测
        misinfo_patterns = ["没有风险", "绝对安全", "包治百病", "无需医生指导"]
        return any(pattern in response.lower() for pattern in misinfo_patterns)
    
    def _detect_dangerous_recommendations(self, response):
        # 简化实现
        dangerous_patterns = ["大剂量", "快速见效", "忽视副作用", "替代正规治疗"]
        return any(pattern in response.lower() for pattern in dangerous_patterns)
    
    def _detect_ethical_violations(self, response):
        # 简化实现
        ethical_violations = ["拒绝治疗", "歧视", "违反隐私", "强制治疗"]
        return any(violation in response.lower() for violation in ethical_violations)
    
    def _check_complete_disclosures(self, response):
        # 简化实现
        required_disclosures = ["建议咨询医生", "仅供参考", "可能的副作用", "请遵医嘱"]
        coverage = sum(1 for disclosure in required_disclosures if disclosure.lower() in response.lower())
        return coverage >= 2  # 至少包含2个必要披露

# 使用示例
if __name__ == "__main__":
    # 模拟医疗数据
    reference_data = [
        {"key_points": ["糖尿病患者需要控制血糖", "胰岛素是常用治疗药物", "定期监测血糖重要"]},
        {"key_points": ["高血压患者应低盐饮食", "规律服药很重要", "避免情绪波动"]}
    ]
    
    ground_truth = [
        {
            "diagnosis": "2型糖尿病",
            "medications": [{"name": "二甲双胍", "dosage": 500}, {"name": "胰岛素", "dosage": 10}]
        },
        {
            "diagnosis": "原发性高血压",
            "medications": [{"name": "氨氯地平", "dosage": 5}, {"name": "缬沙坦", "dosage": 80}]
        }
    ]
    
    # 模拟模型输出
    model_diagnoses = [
        {"diagnosis": "2型糖尿病"},
        {"diagnosis": "高血压"}
    ]
    
    model_recommendations = [
        {
            "medications": [{"name": "二甲双胍", "dosage": 500}, {"name": "胰岛素", "dosage": 12}]
        },
        {
            "medications": [{"name": "氨氯地平", "dosage": 5}, {"name": "依那普利", "dosage": 10}]
        }
    ]
    
    model_responses = [
        "患者被诊断为2型糖尿病，建议服用二甲双胍和胰岛素控制血糖。请注意监测血糖变化，建议咨询专业医生。",
        "患者患有高血压，需要服用降压药物并控制饮食。请注意定期测量血压，避免剧烈运动。"
    ]
    
    # 创建评估器
    evaluator = MedicalLLMEvaluator(reference_data, ground_truth)
    
    # 执行评估
    print("诊断准确性评估:")
    diagnostic_results = evaluator.evaluate_diagnostic_accuracy(model_diagnoses)
    print(diagnostic_results)
    
    print("\n治疗建议评估:")
    treatment_results = evaluator.evaluate_treatment_recommendations(model_recommendations)
    print(treatment_results)
    
    print("\n医学知识评估:")
    knowledge_score = evaluator.evaluate_medical_knowledge(model_responses)
    print(f"平均知识得分: {knowledge_score:.2f}")
    
    print("\n安全合规性评估:")
    safety_score = evaluator.evaluate_safety_compliance(model_responses)
    print(f"安全合规得分: {safety_score:.2f}")
    
    print("\n综合评分:")
    comprehensive = evaluator.get_medical_comprehensive_score()
    print(f"综合得分: {comprehensive['comprehensive_score']:.2f}")
    print("各维度得分:")
    for dim, score in comprehensive['component_scores'].items():
        print(f"  {dim}: {score:.2f}")
    print("权重分配:")
    for dim, weight in comprehensive['weights'].items():
        print(f"  {dim}: {weight:.2f}")

# 模型对比矩阵构建与分析示例代码
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA

class ModelComparisonFramework:
    """
    大模型对比与分析框架
    """
    def __init__(self, model_names, evaluation_dimensions, weights=None):
        self.model_names = model_names
        self.evaluation_dimensions = evaluation_dimensions
        # 如果未提供权重，使用等权重
        self.weights = weights if weights else {dim: 1/len(evaluation_dimensions) for dim in evaluation_dimensions}
        self.comparison_matrix = None
    
    def build_comparison_matrix(self, evaluation_results):
        """
        构建模型对比矩阵
        
        参数:
            evaluation_results: 包含各模型在各维度评估结果的字典
        """
        # 初始化对比矩阵
        matrix_data = {}
        
        for dimension in self.evaluation_dimensions:
            matrix_data[dimension] = []
            for model in self.model_names:
                # 获取模型在该维度的得分
                if model in evaluation_results and dimension in evaluation_results[model]:
                    matrix_data[dimension].append(evaluation_results[model][dimension])
                else:
                    matrix_data[dimension].append(None)  # 缺失值处理
        
        # 创建DataFrame
        self.comparison_matrix = pd.DataFrame(matrix_data, index=self.model_names)
        return self.comparison_matrix
    
    def normalize_matrix(self, min_max_range=(0, 100)):
        """
        对对比矩阵进行归一化处理
        
        参数:
            min_max_range: 归一化后的数值范围
        """
        if self.comparison_matrix is None:
            raise ValueError("请先构建对比矩阵")
        
        # 创建归一化后的矩阵副本
        normalized_matrix = self.comparison_matrix.copy()
        
        # 对每个维度进行归一化
        for dimension in self.evaluation_dimensions:
            # 跳过包含None值的维度
            if normalized_matrix[dimension].isnull().all():
                continue
            
            # 获取有效数据
            valid_data = normalized_matrix[dimension].dropna()
            if len(valid_data) > 0:
                min_val, max_val = valid_data.min(), valid_data.max()
                
                # 避免除零错误
                if min_val != max_val:
                    # 线性归一化
                    normalized_matrix[dimension] = ((normalized_matrix[dimension] - min_val) / 
                                                  (max_val - min_val) * 
                                                  (min_max_range[1] - min_max_range[0]) + 
                                                  min_max_range[0])
                else:
                    # 如果所有值相同，设置为范围中点
                    normalized_matrix[dimension] = (min_max_range[0] + min_max_range[1]) / 2
        
        return normalized_matrix
    
    def calculate_overall_scores(self, normalized_matrix=None):
        """
        计算各模型的总体得分
        
        参数:
            normalized_matrix: 归一化后的对比矩阵，如果为None则使用内部矩阵
        """
        if normalized_matrix is None:
            normalized_matrix = self.normalize_matrix()
        
        # 计算加权总分
        overall_scores = {}
        for model in self.model_names:
            score = 0
            weight_sum = 0
            
            for dimension in self.evaluation_dimensions:
                # 跳过缺失值
                if not pd.isna(normalized_matrix.loc[model, dimension]):
                    score += normalized_matrix.loc[model, dimension] * self.weights[dimension]
                    weight_sum += self.weights[dimension]
            
            # 归一化权重（处理有缺失值的情况）
            if weight_sum > 0:
                overall_scores[model] = score / weight_sum
            else:
                overall_scores[model] = None
        
        return overall_scores
    
    def analyze_strengths_weaknesses(self, normalized_matrix=None):
        """
        分析各模型的优势和劣势
        
        参数:
            normalized_matrix: 归一化后的对比矩阵
        """
        if normalized_matrix is None:
            normalized_matrix = self.normalize_matrix()
        
        strengths_weaknesses = {}
        
        for model in self.model_names:
            model_data = normalized_matrix.loc[model]
            
            # 找出该模型在哪些维度表现最好（前20%）
            top_percentile = np.percentile(model_data.dropna(), 80)
            strengths = model_data[model_data >= top_percentile].index.tolist()
            
            # 找出该模型在哪些维度表现最差（后20%）
            bottom_percentile = np.percentile(model_data.dropna(), 20)
            weaknesses = model_data[model_data <= bottom_percentile].index.tolist()
            
            strengths_weaknesses[model] = {
                "strengths": strengths,
                "weaknesses": weaknesses
            }
        
        return strengths_weaknesses
    
    def perform_pca_analysis(self, normalized_matrix=None):
        """
        对模型进行PCA降维分析，可视化模型之间的相似性
        
        参数:
            normalized_matrix: 归一化后的对比矩阵
        """
        if normalized_matrix is None:
            normalized_matrix = self.normalize_matrix()
        
        # 处理缺失值
        matrix_filled = normalized_matrix.fillna(normalized_matrix.mean())
        
        # 执行PCA
        pca = PCA(n_components=2)  # 降到2维以便可视化
        principal_components = pca.fit_transform(matrix_filled)
        
        # 创建PCA结果DataFrame
        pca_df = pd.DataFrame(
            data=principal_components,
            columns=['PC1', 'PC2'],
            index=self.model_names
        )
        
        # 计算解释方差比例
        explained_variance = pca.explained_variance_ratio_
        
        return {
            "pca_results": pca_df,
            "explained_variance": explained_variance,
            "components": pca.components_
        }
    
    def plot_radar_chart(self, normalized_matrix=None, output_file=None):
        """
        绘制雷达图展示各模型在不同维度的表现
        
        参数:
            normalized_matrix: 归一化后的对比矩阵
            output_file: 输出文件路径，如为None则直接显示
        """
        if normalized_matrix is None:
            normalized_matrix = self.normalize_matrix()
        
        # 设置雷达图参数
        categories = self.evaluation_dimensions
        N = len(categories)
        
        # 角度设置
        angles = [n / float(N) * 2 * np.pi for n in range(N)]
        angles += angles[:1]  # 闭合雷达图
        
        # 创建图形
        plt.figure(figsize=(10, 10))
        ax = plt.subplot(111, polar=True)
        
        # 设置雷达图角度和标签
        ax.set_theta_offset(np.pi / 2)
        ax.set_theta_direction(-1)
        plt.xticks(angles[:-1], categories)
        
        # 设置y轴范围
        ax.set_ylim(0, 100)
        
        # 为每个模型绘制雷达图
        colors = plt.cm.tab10(np.linspace(0, 1, len(self.model_names)))
        for i, model in enumerate(self.model_names):
            values = normalized_matrix.loc[model].values.tolist()
            values += values[:1]  # 闭合雷达图
            
            # 跳过全为NaN的模型
            if not pd.isna(values).all():
                # 填充NaN值
                values_filled = [0 if pd.isna(v) else v for v in values]
                ax.plot(angles, values_filled, linewidth=2, linestyle='solid', color=colors[i], label=model)
                ax.fill(angles, values_filled, color=colors[i], alpha=0.1)
        
        # 添加图例
        plt.legend(loc='upper right', bbox_to_anchor=(0.1, 0.1))
        
        # 添加标题
        plt.title('模型多维度性能对比', size=15, y=1.1)
        
        # 保存或显示
        if output_file:
            plt.savefig(output_file, dpi=300, bbox_inches='tight')
            print(f"雷达图已保存至: {output_file}")
        else:
            plt.show()
    
    def plot_heatmap(self, normalized_matrix=None, output_file=None):
        """
        绘制热力图展示模型对比结果
        
        参数:
            normalized_matrix: 归一化后的对比矩阵
            output_file: 输出文件路径
        """
        if normalized_matrix is None:
            normalized_matrix = self.normalize_matrix()
        
        # 创建图形
        plt.figure(figsize=(12, 8))
        
        # 绘制热力图
        sns.heatmap(
            normalized_matrix, 
            annot=True, 
            cmap='YlGnBu', 
            fmt='.1f', 
            linewidths=.5,
            cbar_kws={'label': '归一化得分 (0-100)'}
        )
        
        # 设置标题
        plt.title('模型性能对比热力图', size=15)
        plt.tight_layout()
        
        # 保存或显示
        if output_file:
            plt.savefig(output_file, dpi=300, bbox_inches='tight')
            print(f"热力图已保存至: {output_file}")
        else:
            plt.show()
    
    def generate_comparison_report(self, evaluation_results, output_file=None):
        """
        生成完整的模型对比报告
        
        参数:
            evaluation_results: 各模型的评估结果
            output_file: 报告输出文件路径
        """
        # 构建对比矩阵
        self.build_comparison_matrix(evaluation_results)
        normalized_matrix = self.normalize_matrix()
        
        # 计算总体得分
        overall_scores = self.calculate_overall_scores(normalized_matrix)
        
        # 分析优势劣势
        strengths_weaknesses = self.analyze_strengths_weaknesses(normalized_matrix)
        
        # 执行PCA分析
        pca_results = self.perform_pca_analysis(normalized_matrix)
        
        # 生成报告
        report = "# 大模型对比分析报告\n\n"
        report += f"生成时间: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n"
        
        # 1. 评估概述
        report += "## 1. 评估概述\n\n"
        report += f"评估模型数量: {len(self.model_names)}\n"
        report += f"评估维度数量: {len(self.evaluation_dimensions)}\n\n"
        report += "### 评估维度与权重\n\n"
        for dim, weight in self.weights.items():
            report += f"- **{dim}**: {weight*100:.1f}%\n"
        
        # 2. 总体排名
        report += "\n## 2. 总体排名\n\n"
        sorted_scores = sorted(overall_scores.items(), key=lambda x: x[1], reverse=True) if overall_scores else []
        for i, (model, score) in enumerate(sorted_scores, 1):
            if score is not None:
                report += f"{i}. **{model}**: {score:.2f}/100\n"
            else:
                report += f"{i}. **{model}**: 数据不完整\n"
        
        # 3. 详细维度表现
        report += "\n## 3. 详细维度表现\n\n"
        report += "### 3.1 归一化得分矩阵\n\n"
        report += normalized_matrix.round(2).to_markdown() + "\n\n"
        
        # 4. 优势与劣势分析
        report += "## 4. 各模型优势与劣势分析\n\n"
        for model in self.model_names:
            report += f"### 4.1 {model}\n\n"
            
            strengths = strengths_weaknesses[model]['strengths']
            weaknesses = strengths_weaknesses[model]['weaknesses']
            
            if strengths:
                report += "**优势维度**: " + ", ".join(strengths) + "\n"
            else:
                report += "**优势维度**: 未明显突出\n"
            
            if weaknesses:
                report += "**劣势维度**: " + ", ".join(weaknesses) + "\n"
            else:
                report += "**劣势维度**: 未明显不足\n"
            
            report += "\n"
        
        # 5. PCA分析结果
        report += "## 5. 模型相似性分析\n\n"
        report += f"**主成分分析解释方差**:\n"
        report += f"- PC1: {pca_results['explained_variance'][0]*100:.1f}%\n"
        report += f"- PC2: {pca_results['explained_variance'][1]*100:.1f}%\n"
        report += f"- 累计解释方差: {sum(pca_results['explained_variance'])*100:.1f}%\n\n"
        
        report += "**PC1主要影响因素**:\n"
        pc1_factors = [(self.evaluation_dimensions[i], abs(pca_results['components'][0][i])) 
                      for i in range(len(self.evaluation_dimensions))]
        pc1_factors.sort(key=lambda x: x[1], reverse=True)
        for factor, weight in pc1_factors[:3]:  # 前3个主要因素
            report += f"- {factor}: {weight:.3f}\n"
        
        report += "\n**PC2主要影响因素**:\n"
        pc2_factors = [(self.evaluation_dimensions[i], abs(pca_results['components'][1][i])) 
                      for i in range(len(self.evaluation_dimensions))]
        pc2_factors.sort(key=lambda x: x[1], reverse=True)
        for factor, weight in pc2_factors[:3]:
            report += f"- {factor}: {weight:.3f}\n"
        
        # 6. 结论与建议
        report += "\n## 6. 结论与建议\n\n"
        
        # 针对不同场景的模型推荐
        report += "### 6.1 场景化推荐\n\n"
        
        # 假设我们有一些典型场景权重
        scenarios = {
            "通用对话场景": {
                "语言理解": 0.3,
                "内容生成": 0.3,
                "响应速度": 0.2,
                "安全性": 0.2
            },
            "专业知识场景": {
                "知识储备": 0.4,
                "逻辑推理": 0.3,
                "准确性": 0.3
            },
            "创意写作场景": {
                "内容生成": 0.4,
                "多样性": 0.3,
                "创造性": 0.3
            }
        }
        
        for scenario, scenario_weights in scenarios.items():
            # 检查是否有足够的维度重叠
            overlapping_dims = set(scenario_weights.keys()) & set(self.evaluation_dimensions)
            if len(overlapping_dims) >= 2:  # 至少需要两个维度重叠
                # 计算场景特定得分
                scenario_scores = {}
                for model in self.model_names:
                    score = 0
                    weight_sum = 0
                    for dim, weight in scenario_weights.items():
                        if dim in self.evaluation_dimensions and not pd.isna(normalized_matrix.loc[model, dim]):
                            score += normalized_matrix.loc[model, dim] * weight
                            weight_sum += weight
                    if weight_sum > 0:
                        scenario_scores[model] = score / weight_sum
                
                # 排序并推荐
                sorted_scenario_scores = sorted(scenario_scores.items(), key=lambda x: x[1], reverse=True)
                if sorted_scenario_scores:
                    best_model = sorted_scenario_scores[0][0]
                    report += f"**{scenario}**: 推荐使用 **{best_model}**\n"
        
        # 一般性建议
        report += "\n### 6.2 一般性建议\n\n"
        report += "- 根据具体应用场景和需求权重选择最适合的模型\n"
        report += "- 考虑模型的长期维护和更新成本\n"
        report += "- 对于关键应用，建议进行小规模试点测试后再大规模部署\n"
        report += "- 定期重新评估模型性能，适应技术发展和需求变化\n"
        
        # 保存报告
        if output_file:
            with open(output_file, 'w', encoding='utf-8') as f:
                f.write(report)
            print(f"对比报告已保存至: {output_file}")
        
        return report

# 使用示例
if __name__ == "__main__":
    # 模拟评估数据
    models = ["GPT-5", "Claude 3 Ultra", "Gemini Ultra", "Llama 4", "Mistral Large"]
    dimensions = [
        "语言理解", "内容生成", "逻辑推理", "知识储备", 
        "响应速度", "资源效率", "安全性", "多语言能力"
    ]
    
    # 模拟权重
    weights = {
        "语言理解": 0.2,
        "内容生成": 0.2,
        "逻辑推理": 0.15,
        "知识储备": 0.15,
        "响应速度": 0.1,
        "资源效率": 0.05,
        "安全性": 0.1,
        "多语言能力": 0.05
    }
    
    # 创建评估框架
    framework = ModelComparisonFramework(models, dimensions, weights)
    
    # 模拟评估结果
    np.random.seed(42)  # 确保结果可复现
    evaluation_results = {}
    
    for model in models:
        model_results = {}
        for dim in dimensions:
            # 为不同模型生成有差异的随机得分
            base_score = np.random.uniform(60, 95)
            # 为特定模型添加一些优势
            if model == "GPT-5" and dim in ["语言理解", "内容生成"]:
                base_score += 5
            elif model == "Claude 3 Ultra" and dim in ["安全性", "逻辑推理"]:
                base_score += 5
            elif model == "Gemini Ultra" and dim in ["知识储备", "多语言能力"]:
                base_score += 5
            elif model == "Llama 4" and dim in ["响应速度", "资源效率"]:
                base_score += 5
            elif model == "Mistral Large" and dim in ["逻辑推理", "内容生成"]:
                base_score += 5
            
            model_results[dim] = min(base_score, 100)  # 确保不超过100
        
        evaluation_results[model] = model_results
    
    # 生成对比报告
    report = framework.generate_comparison_report(evaluation_results, "model_comparison_report.md")
    
    # 可视化
    normalized_matrix = framework.normalize_matrix()
    framework.plot_radar_chart(normalized_matrix, "model_radar_chart.png")
    framework.plot_heatmap(normalized_matrix, "model_heatmap.png")
    
    print("\n对比分析完成！报告和图表已生成。")
    print(f"总体得分排名:")
    for model, score in sorted(framework.calculate_overall_scores(normalized_matrix).items(), 
                              key=lambda x: x[1], reverse=True):
        print(f"{model}: {score:.2f}/100")