2025年计算机视觉三驾马车：物体检测、关键点检测与掩码生成技术全解析

# 2025年基于Transformer的物体检测模型示例实现
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import DetrConfig, DetrModel, DetrForObjectDetection

class AdvancedObjectDetector(nn.Module):
    def __init__(self, num_classes=91, hidden_size=256, num_queries=100,
                 num_hidden_layers=6, num_attention_heads=8,
                 intermediate_size=2048, dropout_rate=0.1):
        super().__init__()
        # 创建DETR配置
        config = DetrConfig(
            hidden_size=hidden_size,
            num_queries=num_queries,
            num_hidden_layers=num_hidden_layers,
            num_attention_heads=num_attention_heads,
            intermediate_size=intermediate_size,
            num_labels=num_classes,
            hidden_dropout_prob=dropout_rate,
            attention_probs_dropout_prob=dropout_rate
        )
        
        # 初始化DETR模型
        self.detr = DetrForObjectDetection(config)
        
        # 初始化权重
        self._initialize_weights()
        
    def _initialize_weights(self):
        # 初始化模型权重
        for m in self.modules():
            if isinstance(m, (nn.Conv2d, nn.Linear)):
                nn.init.trunc_normal_(m.weight, std=0.02)
                if m.bias is not None:
                    nn.init.zeros_(m.bias)
        
    def forward(self, pixel_values, pixel_mask=None, labels=None):
        # pixel_values: [batch_size, 3, image_height, image_width]
        # pixel_mask: [batch_size, image_height, image_width]
        
        # 获取DETR的输出
        outputs = self.detr(
            pixel_values=pixel_values,
            pixel_mask=pixel_mask,
            labels=labels
        )
        
        return outputs

# 2025年先进关键点检测模型实现
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import DetrConfig

class AdvancedKeypointDetector(nn.Module):
    def __init__(self, num_keypoints=17, hidden_size=256, num_queries=100,
                 num_hidden_layers=6, num_attention_heads=8,
                 intermediate_size=2048, dropout_rate=0.1):
        super().__init__()
        # 创建基于DETR的配置
        config = DetrConfig(
            hidden_size=hidden_size,
            num_queries=num_queries,
            num_hidden_layers=num_hidden_layers,
            num_attention_heads=num_attention_heads,
            intermediate_size=intermediate_size,
            hidden_dropout_prob=dropout_rate,
            attention_probs_dropout_prob=dropout_rate
        )
        
        # 初始化Vision Transformer骨干网络
        from transformers import ViTModel
        self.vit = ViTModel.from_pretrained('google/vit-base-patch16-224')
        
        # 关键点检测头
        self.keypoint_head = nn.Sequential(
            nn.Conv2d(hidden_size, 128, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(128, num_keypoints, kernel_size=1)
        )
        
        # 初始化权重
        self._initialize_weights()
        
    def _initialize_weights(self):
        # 初始化模型权重
        for m in self.keypoint_head.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.trunc_normal_(m.weight, std=0.02)
                if m.bias is not None:
                    nn.init.zeros_(m.bias)
        
    def forward(self, pixel_values, pixel_mask=None):
        # 获取Vision Transformer的输出
        outputs = self.vit(
            pixel_values=pixel_values,
            attention_mask=pixel_mask
        )
        
        # 重塑特征图
        batch_size, seq_len, hidden_size = outputs.last_hidden_state.shape
        height = width = int((seq_len - 1) ** 0.5)  # 减1是因为有CLS标记
        
        # 重塑特征，移除CLS标记
        features = outputs.last_hidden_state[:, 1:, :].permute(0, 2, 1).reshape(batch_size, hidden_size, height, width)
        
        # 上采样到原始图像大小的一半
        features = F.interpolate(features, scale_factor=2, mode='bilinear', align_corners=False)
        
        # 通过关键点检测头
        heatmaps = self.keypoint_head(features)
        
        return heatmaps

# 2025年先进掩码生成模型实现
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import SamModel, SamProcessor

class AdvancedMaskGenerator(nn.Module):
    def __init__(self, pretrained_model_name="meta-llama/SAM-3"):
        super().__init__()
        # 加载预训练的SAM模型
        self.sam_model = SamModel.from_pretrained(pretrained_model_name)
        
        # 初始化处理器
        self.processor = SamProcessor.from_pretrained(pretrained_model_name)
        
        # 冻结模型参数
        for param in self.sam_model.parameters():
            param.requires_grad = False
        
    def forward(self, images, input_points=None, input_boxes=None, input_labels=None):
        # 预处理输入
        inputs = self.processor(
            images=images,
            input_points=input_points,
            input_boxes=input_boxes,
            input_labels=input_labels,
            return_tensors="pt"
        )
        
        # 生成掩码
        with torch.no_grad():
            outputs = self.sam_model(
                pixel_values=inputs.pixel_values,
                input_points=inputs.input_points,
                input_boxes=inputs.input_boxes,
                input_labels=inputs.input_labels,
                multimask_output=True
            )
        
        # 后处理掩码
        masks = outputs.pred_masks.squeeze(1)
        scores = outputs.iou_scores
        
        return masks, scores
    
    def generate_mask_from_points(self, image, points, labels=None):
        # 从点提示生成掩码
        inputs = self.processor(
            images=image,
            input_points=[points],
            input_labels=[labels] if labels is not None else None,
            return_tensors="pt"
        )
        
        with torch.no_grad():
            outputs = self.sam_model(
                pixel_values=inputs.pixel_values,
                input_points=inputs.input_points,
                input_labels=inputs.input_labels,
                multimask_output=True
            )
        
        # 选择最佳掩码
        best_mask_idx = outputs.iou_scores.argmax()
        mask = outputs.pred_masks[0, best_mask_idx].cpu().numpy()
        
        return mask
    
    def generate_mask_from_box(self, image, box):
        # 从边界框生成掩码
        inputs = self.processor(
            images=image,
            input_boxes=[[box]],
            return_tensors="pt"
        )
        
        with torch.no_grad():
            outputs = self.sam_model(
                pixel_values=inputs.pixel_values,
                input_boxes=inputs.input_boxes,
                multimask_output=True
            )
        
        # 选择最佳掩码
        best_mask_idx = outputs.iou_scores.argmax()
        mask = outputs.pred_masks[0, best_mask_idx].cpu().numpy()
        
        return mask
    
    def generate_mask_from_text(self, image, text_prompt):
        # 从文本提示生成掩码
        # 注：2025年的SAM-3模型已经支持直接文本提示
        # 这里使用简化实现
        raise NotImplementedError("2025年的完整实现请参考meta-llama/SAM-3官方文档")

# 自动驾驶中的计算机视觉技术协同示例
import torch
from PIL import Image
import numpy as np

class AutonomousDrivingVisionSystem:
    def __init__(self):
        # 初始化三大视觉任务模型
        self.object_detector = AdvancedObjectDetector()
        self.keypoint_detector = AdvancedKeypointDetector()
        self.mask_generator = AdvancedMaskGenerator()
        
    def process_frame(self, frame):
        # 1. 物体检测：识别车辆、行人、交通标志等
        # ... 物体检测处理代码 ...
        
        # 2. 关键点检测：识别车道线、交通标志关键点等
        # ... 关键点检测处理代码 ...
        
        # 3. 掩码生成：分割道路、障碍物等
        # ... 掩码生成处理代码 ...
        
        # 4. 信息融合：综合三种技术的结果，生成完整的环境感知结果
        # ... 信息融合处理代码 ...
        
        return {
            'objects': detected_objects,
            'keypoints': detected_keypoints,
            'masks': generated_masks,
            'fused_result': fused_environment_info
        }

# 计算机视觉模型知识蒸馏示例
import torch
import torch.nn as nn
import torch.nn.functional as F

class KeypointDistillationTrainer:
    def __init__(self, teacher_model, student_model, num_keypoints=17):
        # 初始化教师模型（大型高精度模型）
        self.teacher_model = teacher_model
        for param in self.teacher_model.parameters():
            param.requires_grad = False  # 冻结教师模型参数
        
        # 初始化学生模型（小型高效模型）
        self.student_model = student_model
        
        # 定义优化器
        self.optimizer = torch.optim.AdamW(
            self.student_model.parameters(),
            lr=1e-4,
            weight_decay=1e-4
        )
        
        # 定义温度参数（控制软标签的平滑程度）
        self.temperature = 2.0
        
        # 定义损失权重
        self.heatmap_weight = 0.7  # 热力图损失权重
        self.distill_weight = 0.3  # 知识蒸馏损失权重
        
        # 定义损失函数
        self.criterion = nn.MSELoss()
        
    def compute_distillation_loss(self, teacher_heatmaps, student_heatmaps, temperature):
        # 将热力图转换为概率分布
        teacher_probs = F.softmax(teacher_heatmaps / temperature, dim=1)
        student_log_probs = F.log_softmax(student_heatmaps / temperature, dim=1)
        
        # 使用KL散度计算蒸馏损失
        distillation_loss = F.kl_div(
            student_log_probs,
            teacher_probs,
            reduction='batchmean'
        ) * (temperature ** 2)  # 缩放损失以保持梯度大小
        
        return distillation_loss
    
    def train_step(self, images, targets):
        # 启用教师模型的推理模式
        with torch.no_grad():
            teacher_heatmaps = self.teacher_model(images)
        
        # 学生模型前向传播
        student_heatmaps = self.student_model(images)
        
        # 计算原始热力图损失
        heatmap_loss = self.criterion(student_heatmaps, targets)
        
        # 计算知识蒸馏损失
        distillation_loss = self.compute_distillation_loss(
            teacher_heatmaps=teacher_heatmaps,
            student_heatmaps=student_heatmaps,
            temperature=self.temperature
        )
        
        # 计算总损失
        total_loss = (self.heatmap_weight * heatmap_loss) + (self.distill_weight * distillation_loss)
        
        # 反向传播和参数更新
        self.optimizer.zero_grad()
        total_loss.backward()
        self.optimizer.step()
        
        return {
            'total_loss': total_loss.item(),
            'heatmap_loss': heatmap_loss.item(),
            'distillation_loss': distillation_loss.item()
        }
    
    def train_epoch(self, dataloader, epoch):
        self.student_model.train()
        total_loss = 0.0
        
        for batch_idx, (images, targets) in enumerate(dataloader):
            # 执行训练步骤
            loss_dict = self.train_step(images, targets)
            total_loss += loss_dict['total_loss']
            
            # 打印训练进度
            if batch_idx % 100 == 0:
                print(f"Epoch {epoch}, Batch {batch_idx}, Loss: {loss_dict['total_loss']:.4f}")
        
        return total_loss / len(dataloader)

# 模型量化示例
import torch
from transformers import AutoModelForObjectDetection, AutoProcessor

class QuantizedVisionModel:
    def __init__(self, model_name, quantize_type="int8"):
        # 加载原始模型
        self.model = AutoModelForObjectDetection.from_pretrained(model_name)
        self.processor = AutoProcessor.from_pretrained(model_name)
        
        # 进行模型量化
        self.quantized_model = torch.quantization.quantize_dynamic(
            self.model,
            {torch.nn.Linear, torch.nn.Conv2d},
            dtype=torch.qint8 if quantize_type == "int8" else torch.quint8
        )
        
    def inference(self, image):
        # 预处理图像
        inputs = self.processor(images=image, return_tensors="pt")
        
        # 使用量化模型进行推理
        with torch.no_grad():
            outputs = self.quantized_model(**inputs)
        
        # 后处理结果
        # ... 后处理代码 ...
        
        return outputs