AI模型可解释性：2025年最新技术进展与实践指南

timeline
    title 可解释AI的发展历程
    2010前 : 简单模型可解释性
    2010-2018 : 复杂模型事后解释
    2018-2024 : 全面可解释性研究
    2024至今 : 大型语言模型可解释性

radarChart
    title LLM可解释性与传统模型可解释性的比较
    xAxis [参数规模, 结构复杂度, 知识隐含性, 决策透明度, 涌现能力]
    yAxis 0-100
    A[LLM可解释性] 95, 90, 85, 30, 80
    B[传统模型可解释性] 30, 40, 50, 80, 20

# 基于梯度的特征重要性分析示例
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

def gradient_based_importance(model, tokenizer, text, target_token=None):
    # 标记化输入
    inputs = tokenizer(text, return_tensors="pt")
    input_ids = inputs.input_ids.clone()
    input_ids.requires_grad = True
    
    # 获取模型输出
    outputs = model(input_ids=input_ids)
    logits = outputs.logits
    
    # 选择目标token（如果未指定，则选择最后一个token）
    if target_token is None:
        target_pos = -1
    else:
        # 查找目标token的位置
        target_ids = tokenizer(target_token, add_special_tokens=False).input_ids
        if len(target_ids) != 1:
            raise ValueError("Target token must be a single token")
        target_pos = (input_ids[0] == target_ids[0]).nonzero().item()
    
    # 计算梯度
    target_logit = logits[0, target_pos, input_ids[0, target_pos]]
    target_logit.backward()
    
    # 计算梯度范数作为重要性分数
    gradients = input_ids.grad.data.abs().squeeze().tolist()
    
    # 获取输入的token
    tokens = tokenizer.convert_ids_to_tokens(input_ids[0].tolist())
    
    # 创建重要性分数列表
    importance_scores = [(token, score) for token, score in zip(tokens, gradients)]
    
    # 按重要性排序
    importance_scores.sort(key=lambda x: x[1], reverse=True)
    
    return importance_scores

# 注意力可视化示例
import torch
from transformers import AutoTokenizer, AutoModel
import matplotlib.pyplot as plt
import seaborn as sns

def visualize_attention(model, tokenizer, text, layer=0, head=0):
    # 标记化输入
    inputs = tokenizer(text, return_tensors="pt")
    
    # 获取模型输出和注意力权重
    with torch.no_grad():
        outputs = model(**inputs, output_attentions=True)
    
    # 获取指定层和头的注意力权重
    attention = outputs.attentions[layer][0, head].cpu().numpy()
    
    # 获取输入的token
    tokens = tokenizer.convert_ids_to_tokens(inputs.input_ids[0].tolist())
    
    # 创建热力图
    plt.figure(figsize=(10, 10))
    sns.heatmap(attention, xticklabels=tokens, yticklabels=tokens, cmap='viridis')
    plt.title(f'Attention Layer {layer}, Head {head}')
    plt.tight_layout()
    
    # 保存图像
    plt.savefig('attention_visualization.png')
    plt.close()
    
    return 'attention_visualization.png'

# 概念提取示例
import torch
from transformers import AutoTokenizer, AutoModel
from sklearn.cluster import KMeans
import numpy as np

def extract_concepts(model, tokenizer, texts, layer=0, n_clusters=10):
    all_activations = []
    all_tokens = []
    
    # 获取所有文本的激活
    for text in texts:
        inputs = tokenizer(text, return_tensors="pt", truncation=True)
        with torch.no_grad():
            outputs = model(**inputs, output_hidden_states=True)
        
        # 获取指定层的隐藏状态
        activations = outputs.hidden_states[layer].squeeze().cpu().numpy()
        tokens = tokenizer.convert_ids_to_tokens(inputs.input_ids[0].tolist())
        
        all_activations.append(activations)
        all_tokens.extend(tokens)
    
    # 合并所有激活
    all_activations = np.vstack(all_activations)
    
    # 使用K-means聚类
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    clusters = kmeans.fit_predict(all_activations)
    
    # 收集每个聚类中的token
    concept_tokens = {i: [] for i in range(n_clusters)}
    for token, cluster in zip(all_tokens, clusters):
        concept_tokens[cluster].append(token)
    
    # 对每个聚类的token进行排序，选择出现频率高的token作为概念代表
    concepts = {}
    for cluster_id, tokens in concept_tokens.items():
        # 统计token频率
        token_counts = {}
        for token in tokens:
            if token in token_counts:
                token_counts[token] += 1
            else:
                token_counts[token] = 1
        
        # 按频率排序
        sorted_tokens = sorted(token_counts.items(), key=lambda x: x[1], reverse=True)
        
        # 选择前5个token作为概念代表
        concept_name = "、".join([token for token, _ in sorted_tokens[:5]])
        concepts[cluster_id] = concept_name
    
    return concepts

# 交互式输入变体探索示例
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

class InteractiveExplorer:
    def __init__(self, model_name):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
        self.model = AutoModelForCausalLM.from_pretrained(model_name)
        
    def explore_input_variations(self, base_text, variations):
        results = []
        
        # 测试基础文本
        base_output = self._generate_text(base_text)
        results.append({
            'type': 'base',
            'input': base_text,
            'output': base_output
        })
        
        # 测试变体
        for i, variation in enumerate(variations):
            var_output = self._generate_text(variation)
            results.append({
                'type': 'variation',
                'id': i+1,
                'input': variation,
                'output': var_output,
                'difference': self._calculate_difference(base_output, var_output)
            })
        
        return results
    
    def _generate_text(self, text, max_length=100):
        inputs = self.tokenizer(text, return_tensors="pt")
        with torch.no_grad():
            outputs = self.model.generate(
                input_ids=inputs.input_ids,
                attention_mask=inputs.attention_mask,
                max_length=max_length,
                do_sample=True,
                temperature=0.7
            )
        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
    
    def _calculate_difference(self, text1, text2):
        # 简单的文本差异计算，实际应用中可以使用更复杂的方法
        words1 = set(text1.lower().split())
        words2 = set(text2.lower().split())
        common_words = words1.intersection(words2)
        return len(common_words) / (len(words1) + len(words2) - len(common_words)) if (len(words1) + len(words2) - len(common_words)) > 0 else 0

# 解释准确性评估示例
import torch
from sklearn.metrics import accuracy_score
from transformers import AutoTokenizer, AutoModelForCausalLM

def evaluate_explanation_fidelity(model, tokenizer, explain_func, test_data, importance_threshold=0.5):
    correct_predictions = 0
    total_predictions = 0
    
    for text, label in test_data:
        # 获取原始预测
        inputs = tokenizer(text, return_tensors="pt")
        with torch.no_grad():
            outputs = model(**inputs)
            logits = outputs.logits
            original_pred = torch.argmax(logits[0, -1, :]).item()
        
        # 获取特征重要性
        importance_scores = explain_func(model, tokenizer, text)
        
        # 保留重要性高于阈值的特征
        important_tokens = [token for token, score in importance_scores if score >= importance_threshold]
        important_token_ids = tokenizer.convert_tokens_to_ids(important_tokens)
        
        # 创建掩码文本，只保留重要特征
        masked_input_ids = []
        original_input_ids = inputs.input_ids[0].tolist()
        for token_id in original_input_ids:
            if token_id in important_token_ids or token_id in [tokenizer.cls_token_id, tokenizer.sep_token_id]:
                masked_input_ids.append(token_id)
            else:
                masked_input_ids.append(tokenizer.mask_token_id)
        
        # 重新生成文本
        masked_inputs = {"input_ids": torch.tensor([masked_input_ids])}
        with torch.no_grad():
            masked_outputs = model(**masked_inputs)
            masked_logits = masked_outputs.logits
            masked_pred = torch.argmax(masked_logits[0, -1, :]).item()
        
        # 计算保真度
        if original_pred == masked_pred:
            correct_predictions += 1
        total_predictions += 1
    
    # 计算保真度分数
    fidelity_score = correct_predictions / total_predictions
    
    return fidelity_score

# 创建并激活虚拟环境
conda create -n interpretablellm python=3.9
conda activate interpretablellm

# 安装必要的依赖
pip install torch torchvision torchaudio
pip install transformers datasets evaluate
pip install sentencepiece pillow
pip install accelerate bitsandbytes
pip install huggingface_hub
pip install scikit-learn scipy
pip install shap lime
pip install matplotlib seaborn plotly
pip install flask fastapi uvicorn  # 用于部署API服务

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from interpretablellm import InterpretableLLMFramework, ExplanationConfig

# 加载预训练模型和分词器
model_name = "mistralai/Mistral-7B-v0.1"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.float16,
    device_map="auto"
)

# 创建InterpretableLLM框架实例
interpretable_framework = InterpretableLLMFramework()

# 配置解释方法
explanation_config = ExplanationConfig(
    # 局部解释配置
    local_explanation={
        'enabled': True,
        'methods': [
            'feature_importance',
            'attention_visualization',
            'counterfactual_explanation',
            'reasoning_chain'
        ],
        'params': {
            'feature_importance': {'method': 'gradient_based'},
            'attention_visualization': {'layer': 0, 'head': 0},
            'counterfactual_explanation': {'num_samples': 5},
            'reasoning_chain': {'max_steps': 10}
        }
    },
    # 全局解释配置
    global_explanation={
        'enabled': False,
        'methods': ['concept_extraction', 'bias_detection'],
        'params': {
            'concept_extraction': {'n_clusters': 10},
            'bias_detection': {'protected_attributes': ['gender', 'age']}
        }
    },
    # 交互式解释配置
    interactive_explanation={
        'enabled': False,
        'methods': ['exploratory_analysis', 'hypothesis_testing']
    },
    # 可视化配置
    visualization={
        'enabled': True,
        'format': 'html',
        'save_path': './explanations/'
    }
)

# 生成解释
def generate_explanation(text):
    # 使用InterpretableLLM框架生成解释
    result = interpretable_framework.explain(
        model=model,
        tokenizer=tokenizer,
        text=text,
        config=explanation_config
    )
    
    # 获取结果
    prediction = result['prediction']
    local_explanations = result['local_explanations']
    global_explanations = result.get('global_explanations', {})
    visualizations = result.get('visualizations', {})
    
    return prediction, local_explanations, global_explanations, visualizations

# 测试示例
text = "请解释什么是AI模型可解释性以及它的重要性。"
prediction, local_explanations, global_explanations, visualizations = generate_explanation(text)

print(f"输入: {text}")
print(f"预测: {prediction}")
print(f"局部解释: {local_explanations.keys()}")
print(f"全局解释: {global_explanations.keys()}")
print(f"可视化: {visualizations.keys()}")

# 打印特征重要性示例
if 'feature_importance' in local_explanations:
    print("\n特征重要性:")
    for token, importance in local_explanations['feature_importance'][:5]:
        print(f"  {token}: {importance:.4f}")

# 打印反事实解释示例
if 'counterfactual_explanation' in local_explanations:
    print("\n反事实解释:")
    for i, cf in enumerate(local_explanations['counterfactual_explanation'][:2]):
        print(f"  反事实样本 {i+1}:")
        print(f"    输入: {cf['input']}")
        print(f"    输出: {cf['output']}")
        print(f"    变化: {cf['changes']}")

from interpretablellm import InterpretableLLMFramework, BaseExplanationMethod

# 创建自定义解释方法
class MyCustomExplanation(BaseExplanationMethod):
    def __init__(self, config):
        super().__init__(config)
        self.param1 = config.get('param1', 0.5)
        self.param2 = config.get('param2', 'default')
    
    def explain(self, model, tokenizer, text, **kwargs):
        # 实现自定义解释逻辑
        # 这里是一个简化的示例
        explanation = {
            'custom_metric': 0.85,
            'custom_insights': ['这是一个自定义解释示例', f'参数1: {self.param1}', f'参数2: {self.param2}'],
            'text_analysis': self._analyze_text(text)
        }
        
        return explanation
    
    def _analyze_text(self, text):
        # 简单的文本分析示例
        words = text.lower().split()
        word_count = len(words)
        unique_words = len(set(words))
        
        return {
            'word_count': word_count,
            'unique_words': unique_words,
            'lexical_diversity': unique_words / word_count if word_count > 0 else 0
        }

# 创建InterpretableLLM框架实例
interpretable_framework = InterpretableLLMFramework()

# 注册自定义解释方法
interpretable_framework.register_explanation_method('custom_explanation', MyCustomExplanation)

# 配置解释方法，包含自定义方法
explanation_config = ExplanationConfig(
    local_explanation={
        'enabled': True,
        'methods': ['custom_explanation'],
        'params': {
            'custom_explanation': {
                'param1': 0.7,
                'param2': 'custom_value'
            }
        }
    }
)

# 使用自定义解释方法
# prediction, local_explanations, _, _ = interpretable_framework.explain(
#     model=model,
#     tokenizer=tokenizer,
#     text=text,
#     config=explanation_config
# )
# 
# if 'custom_explanation' in local_explanations:
#     print("自定义解释结果:")
#     print(local_explanations['custom_explanation'])

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import uvicorn
from interpretablellm import InterpretableLLMFramework

# 创建FastAPI应用
app = FastAPI(title="InterpretableLLM API", description="大型语言模型可解释性API")

# 加载InterpretableLLM框架（在实际应用中，应该在应用启动时加载）
# interpretable_framework = InterpretableLLMFramework.from_config("path/to/config.json")
# 这里我们模拟一个已加载的框架实例
class MockInterpretableFramework:
    def explain(self, text, **kwargs):
        # 模拟解释结果
        return {
            'prediction': f"这是对'{text}'的预测结果。",
            'local_explanations': {
                'feature_importance': [("AI", 0.95), ("模型", 0.85), ("可解释性", 0.90)],
                'attention_visualization': "attention_heatmap.png"
            },
            'global_explanations': {},
            'visualizations': {}
        }

interpretable_framework = MockInterpretableFramework()

# 定义请求和响应模型
class ExplainRequest(BaseModel):
    text: str
    include_local: bool = True
    include_global: bool = False
    include_visualizations: bool = True

class ExplainResponse(BaseModel):
    prediction: str
    local_explanations: dict
    global_explanations: dict
    visualizations: dict

# 定义解释端点
@app.post("/explain", response_model=ExplainResponse)
def explain(request: ExplainRequest):
    try:
        # 生成解释
        result = interpretable_framework.explain(
            text=request.text,
            include_local=request.include_local,
            include_global=request.include_global,
            include_visualizations=request.include_visualizations
        )
        
        # 返回结果
        return ExplainResponse(
            prediction=result['prediction'],
            local_explanations=result.get('local_explanations', {}),
            global_explanations=result.get('global_explanations', {}),
            visualizations=result.get('visualizations', {})
        )
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

# 定义健康检查端点
@app.get("/health")
def health_check():
    return {"status": "healthy"}

# 运行API服务（在实际应用中，应该使用uvicorn命令行运行）
if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8000)