SmolVLM：多模态小型模型的设计与实现

mindmap
root((SmolVLM))
    模型概述
    融合技术
    架构设计
    训练方法
    实验结果
    部署应用
    代码示例
    创新价值
    未来趋势

import torch
import torch.nn as nn

class DepthwiseSeparableConv(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1):
        super().__init__()
        # 深度卷积
        self.depthwise = nn.Conv2d(
            in_channels, in_channels, kernel_size, stride, padding, groups=in_channels, bias=False
        )
        # 逐点卷积
        self.pointwise = nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
        # 批归一化和激活函数
        self.bn = nn.BatchNorm2d(out_channels)
        self.relu = nn.ReLU6(inplace=True)
    
    def forward(self, x):
        x = self.depthwise(x)
        x = self.pointwise(x)
        x = self.bn(x)
        x = self.relu(x)
        return x

class LightweightVisionEncoder(nn.Module):
    def __init__(self, input_channels=3, embedding_dim=512):
        super().__init__()
        # 初始卷积层
        self.stem = nn.Sequential(
            nn.Conv2d(input_channels, 16, kernel_size=3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(16),
            nn.ReLU6(inplace=True)
        )
        
        # 深度可分离卷积块
        self.blocks = nn.Sequential(
            DepthwiseSeparableConv(16, 24, stride=2),
            DepthwiseSeparableConv(24, 32, stride=2),
            DepthwiseSeparableConv(32, 64, stride=2),
            DepthwiseSeparableConv(64, 96, stride=1),
            DepthwiseSeparableConv(96, 160, stride=2),
            DepthwiseSeparableConv(160, 320, stride=1)
        )
        
        # 输出投影层
        self.projection = nn.Sequential(
            nn.AdaptiveAvgPool2d((1, 1)),
            nn.Flatten(),
            nn.Linear(320, embedding_dim),
            nn.LayerNorm(embedding_dim)
        )
    
    def forward(self, x):
        # 输入形状: [batch_size, 3, height, width]
        x = self.stem(x)
        x = self.blocks(x)
        x = self.projection(x)
        # 输出形状: [batch_size, embedding_dim]
        return x

import torch
import torch.nn as nn
from transformers.models.llama.modeling_llama import LlamaAttention, LlamaMLP, LlamaRMSNorm

class SmallLanguageModel(nn.Module):
    def __init__(self, vocab_size=32000, hidden_size=768, num_hidden_layers=6,
                 num_attention_heads=12, intermediate_size=3072, max_position_embeddings=2048):
        super().__init__()
        # 词嵌入层
        self.embed_tokens = nn.Embedding(vocab_size, hidden_size, padding_idx=0)
        # 位置编码
        self.embed_positions = nn.Embedding(max_position_embeddings, hidden_size)
        # 注意力层
        self.attention_layers = nn.ModuleList([
            LlamaAttention(hidden_size, num_attention_heads, bias=False)
            for _ in range(num_hidden_layers)
        ])
        # 前馈网络层
        self.feed_forward_layers = nn.ModuleList([
            LlamaMLP(hidden_size, intermediate_size, hidden_size, bias=False)
            for _ in range(num_hidden_layers)
        ])
        # 归一化层
        self.input_layernorms = nn.ModuleList([
            LlamaRMSNorm(hidden_size, eps=1e-6)
            for _ in range(num_hidden_layers)
        ])
        self.post_attention_layernorms = nn.ModuleList([
            LlamaRMSNorm(hidden_size, eps=1e-6)
            for _ in range(num_hidden_layers)
        ])
        # 输出层
        self.lm_head = nn.Linear(hidden_size, vocab_size, bias=False)
        
        # 初始化权重
        self.apply(self._init_weights)
    
    def _init_weights(self, module):
        # 初始化权重
        if isinstance(module, (nn.Linear, nn.Embedding)):
            nn.init.normal_(module.weight, mean=0.0, std=0.02)
            if isinstance(module, nn.Linear) and module.bias is not None:
                nn.init.zeros_(module.bias)
        elif isinstance(module, nn.LayerNorm):
            nn.init.zeros_(module.bias)
            nn.init.ones_(module.weight)
    
    def forward(self, input_ids, attention_mask=None):
        # 输入形状: [batch_size, seq_len]
        batch_size, seq_len = input_ids.shape
        
        # 生成位置_ids
        position_ids = torch.arange(0, seq_len, dtype=torch.long, device=input_ids.device)
        position_ids = position_ids.unsqueeze(0).expand(batch_size, seq_len)
        
        # 嵌入层
        x = self.embed_tokens(input_ids) + self.embed_positions(position_ids)
        
        # 注意力掩码处理
        if attention_mask is not None:
            # 将注意力掩码转换为LlamaAttention所需的格式
            attention_mask = attention_mask[:, None, None, :]
            attention_mask = attention_mask.to(dtype=x.dtype)
            attention_mask = (1.0 - attention_mask) * torch.finfo(x.dtype).min
        
        # 处理每一层
        for i in range(len(self.attention_layers)):
            residual = x
            x = self.input_layernorms[i](x)
            # 注意力层
            x, _ = self.attention_layers[i](x, attention_mask=attention_mask)
            x = residual + x
            
            residual = x
            x = self.post_attention_layernorms[i](x)
            # 前馈网络层
            x = self.feed_forward_layers[i](x)
            x = residual + x
        
        # 输出层
        logits = self.lm_head(x)
        
        return logits

import torch
import torch.nn as nn

class EfficientFusionModule(nn.Module):
    def __init__(self, visual_dim, language_dim, hidden_dim, num_heads=4):
        super().__init__()
        # 投影层
        self.visual_proj = nn.Linear(visual_dim, hidden_dim)
        self.language_proj = nn.Linear(language_dim, hidden_dim)
        
        # 融合注意力机制
        self.fusion_attention = nn.MultiheadAttention(
            hidden_dim, num_heads, batch_first=True
        )
        
        # 前馈网络
        self.fusion_mlp = nn.Sequential(
            nn.Linear(hidden_dim * 2, hidden_dim),
            nn.GELU(),
            nn.Linear(hidden_dim, hidden_dim)
        )
        
        # 归一化层
        self.norm1 = nn.LayerNorm(hidden_dim)
        self.norm2 = nn.LayerNorm(hidden_dim)
        
        # 融合权重预测器
        self.weight_predictor = nn.Sequential(
            nn.Linear(visual_dim + language_dim, 64),
            nn.GELU(),
            nn.Linear(64, 2),
            nn.Softmax(dim=-1)
        )
    
    def forward(self, visual_features, language_features):
        # 输入形状: 
        # visual_features: [batch_size, visual_dim]
        # language_features: [batch_size, seq_len, language_dim]
        
        # 投影到相同维度
        visual_proj = self.visual_proj(visual_features)
        language_proj = self.language_proj(language_features)
        
        # 扩展视觉特征维度以匹配语言特征
        visual_proj_expanded = visual_proj.unsqueeze(1).repeat(1, language_proj.shape[1], 1)
        
        # 计算融合权重
        visual_language_cat = torch.cat([
            visual_features.unsqueeze(1).repeat(1, language_features.shape[1], 1),
            language_features
        ], dim=-1)
        fusion_weights = self.weight_predictor(visual_language_cat)
        
        # 融合注意力
        attn_output, _ = self.fusion_attention(
            language_proj,  # query
            visual_proj_expanded,  # key
            visual_proj_expanded,  # value
        )
        
        # 应用融合权重
        weighted_visual = fusion_weights[:, :, 0].unsqueeze(-1) * visual_proj_expanded
        weighted_language = fusion_weights[:, :, 1].unsqueeze(-1) * language_proj
        
        # 融合特征
        fused = weighted_visual + weighted_language + attn_output
        fused = self.norm1(fused)
        
        # 通过MLP进一步融合
        residual = fused
        mlp_input = torch.cat([visual_proj_expanded, language_proj], dim=-1)
        fused = residual + self.fusion_mlp(mlp_input)
        fused = self.norm2(fused)
        
        return fused

import torch
import torch.nn as nn

class SmolVLM(nn.Module):
    def __init__(self, vision_encoder, language_model, fusion_module, 
                 vision_feature_dim=512, language_feature_dim=768, hidden_dim=768):
        super().__init__()
        self.vision_encoder = vision_encoder
        self.language_model = language_model
        self.fusion_module = fusion_module
        
        # 可选的额外投影层（如果需要）
        self.visual_feature_proj = nn.Linear(vision_feature_dim, hidden_dim)
        self.language_feature_proj = nn.Linear(language_feature_dim, hidden_dim)
        
        # 输出层（用于特定任务）
        self.task_output_layer = nn.Linear(hidden_dim, language_model.lm_head.out_features)
    
    def forward(self, images, input_ids, attention_mask=None):
        # 提取视觉特征
        visual_features = self.vision_encoder(images)
        
        # 处理语言输入（不经过语言模型的最终输出层）
        with torch.no_grad():
            # 获取语言模型的隐藏状态
            # 这里假设language_model有一个方法可以返回隐藏状态
            language_hidden_states = self.language_model.get_hidden_states(
                input_ids, attention_mask=attention_mask
            )
        
        # 应用可选的投影
        visual_features = self.visual_feature_proj(visual_features)
        
        # 融合视觉和语言特征
        fused_features = self.fusion_module(visual_features, language_hidden_states)
        
        # 生成输出
        outputs = self.task_output_layer(fused_features)
        
        return outputs
    
    def generate(self, images, input_ids, max_length=100, temperature=0.7):
        # 生成文本响应
        self.eval()
        with torch.no_grad():
            # 提取视觉特征
            visual_features = self.vision_encoder(images)
            
            # 初始化生成结果
            generated = input_ids
            
            # 生成过程
            for _ in range(max_length):
                # 获取当前隐藏状态
                with torch.no_grad():
                    language_hidden_states = self.language_model.get_hidden_states(
                        generated, attention_mask=torch.ones_like(generated)
                    )
                
                # 应用可选的投影
                visual_proj = self.visual_feature_proj(visual_features)
                
                # 融合特征
                fused_features = self.fusion_module(visual_proj, language_hidden_states)
                
                # 获取最后一个位置的输出
                last_output = self.task_output_layer(fused_features[:, -1, :])
                
                # 应用温度调整
                last_output = last_output / temperature
                
                # 采样下一个token
                probs = torch.softmax(last_output, dim=-1)
                next_token = torch.multinomial(probs, num_samples=1)
                
                # 添加到生成结果
                generated = torch.cat([generated, next_token], dim=1)
                
                # 检查是否生成结束符
                if next_token.item() == self.language_model.config.eos_token_id:
                    break
        
        self.train()
        return generated

bar chart
title 各模型在VQAv2和COCO CIDEr上的性能对比
xaxis 模型
yaxis 性能指标
series VQAv2准确率(%) : 76.2, 81.2, 82.5, 65.7, 72.1
series COCO CIDEr : 125.3, 135.7, 137.2, 110.2, 118.5

import torch
from transformers import AutoModelForCausalLM, AutoProcessor

# 加载SmolVLM模型和处理器
model = AutoModelForCausalLM.from_pretrained("smol-vlm/smolvlm-700m")
processor = AutoProcessor.from_pretrained("smol-vlm/smolvlm-700m")

# 转换为TorchScript
model.eval()
with torch.no_grad():
    # 准备示例输入
    image = torch.randn(1, 3, 224, 224)
    input_ids = torch.randint(0, model.config.vocab_size, (1, 10))
    
    # 跟踪模型
    traced_model = torch.jit.trace(model, (image, input_ids))
    
    # 保存TorchScript模型
    torch.jit.save(traced_model, "smolvlm_traced.pt")

# INT8量化
quantized_model = torch.quantization.quantize_dynamic(
    model,
    {torch.nn.Linear},  # 仅量化线性层
    dtype=torch.qint8
)

# 保存量化模型
torch.save(quantized_model.state_dict(), "smolvlm_quantized.pt")

import torch
import cv2
from transformers import AutoModelForCausalLM, AutoProcessor

# 加载量化后的SmolVLM模型
model = AutoModelForCausalLM.from_pretrained(
    "smol-vlm/smolvlm-700m-quantized",
    torch_dtype=torch.float32,
    low_cpu_mem_usage=True
)
model.eval()

# 加载处理器
processor = AutoProcessor.from_pretrained("smol-vlm/smolvlm-700m")

# 初始化摄像头
cap = cv2.VideoCapture(0)

while True:
    # 捕获图像
    ret, frame = cap.read()
    if not ret:
        break
    
    # 显示捕获的图像
    cv2.imshow("Camera", frame)
    
    # 处理按键事件
    key = cv2.waitKey(1) & 0xFF
    
    if key == ord('q'):
        # 退出程序
        break
    elif key == ord('c'):
        # 捕获当前图像并进行推理
        # 调整图像大小
        resized_frame = cv2.resize(frame, (224, 224))
        
        # 准备问题
        prompt = "描述这张图片"
        
        # 处理输入
        inputs = processor(text=prompt, images=resized_frame, return_tensors="pt")
        
        # 进行推理
        with torch.no_grad():
            outputs = model.generate(
                **inputs,
                max_new_tokens=50,
                temperature=0.7,
                do_sample=True
            )
        
        # 解码结果
        response = processor.decode(outputs[0], skip_special_tokens=True)
        print(f"响应: {response}")

# 释放资源
cap.release()
cv2.destroyAllWindows()

// 使用PyTorch Mobile集成SmolVLM
import org.pytorch.IValue;
import org.pytorch.Module;
import org.pytorch.Tensor;
import org.pytorch.torchvision.TensorImageUtils;

public class SmolVLMWrapper {
    private Module model;
    
    public SmolVLMWrapper(String modelPath) {
        // 加载模型
        model = Module.load(modelPath);
    }
    
    public String generateDescription(Bitmap image, String prompt) {
        // 预处理图像
        Bitmap resizedImage = Bitmap.createScaledBitmap(image, 224, 224, true);
        
        // 转换为Tensor
        float[] mean = {0.485f, 0.456f, 0.406f};
        float[] std = {0.229f, 0.224f, 0.225f};
        Tensor inputTensor = TensorImageUtils.bitmapToFloat32Tensor(resizedImage, mean, std);
        inputTensor = inputTensor.unsqueeze(0); // 添加批处理维度
        
        // 处理文本提示（这里简化处理，实际需要文本分词器）
        long[] inputIds = tokenizePrompt(prompt);
        Tensor inputIdsTensor = Tensor.fromBlob(inputIds, new long[]{1, inputIds.length});
        
        // 执行推理
        IValue[] outputs = model.forward(IValue.from(inputTensor), IValue.from(inputIdsTensor));
        
        // 处理输出（这里简化处理，实际需要解码生成的文本）
        Tensor outputTensor = outputs[0].toTensor();
        String description = decodeOutput(outputTensor);
        
        return description;
    }
    
    private long[] tokenizePrompt(String prompt) {
        // 实际实现需要使用与模型匹配的分词器
        // 这里仅作为示例
        return new long[]{1, 2, 3, 4, 5}; // 占位符
    }
    
    private String decodeOutput(Tensor outputTensor) {
        // 实际实现需要使用与模型匹配的解码器
        // 这里仅作为示例
        return "这是一张包含...的图片";
    }
}

pip install torch transformers datasets opencv-python Pillow

import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import requests

# 加载处理器和模型
processor = AutoProcessor.from_pretrained("smol-vlm/smolvlm-700m")
model = AutoModelForCausalLM.from_pretrained("smol-vlm/smolvlm-700m")

# 设置为评估模式
model.eval()

# 加载图像
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

# 准备提示
prompt = "描述这张图片"

# 处理输入
inputs = processor(text=prompt, images=image, return_tensors="pt")

# 生成描述
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=100,
        temperature=0.7,
        do_sample=True
    )

# 解码结果
description = processor.decode(outputs[0], skip_special_tokens=True)
print(f"提示: {prompt}")
print(f"描述: {description}")

import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import requests

# 加载处理器和模型
processor = AutoProcessor.from_pretrained("smol-vlm/smolvlm-700m")
model = AutoModelForCausalLM.from_pretrained("smol-vlm/smolvlm-700m")

# 设置为评估模式
model.eval()

# 加载图像
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
image = Image.open(requests.get(url, stream=True).raw)

# 准备问题
question = "图片中有几只猫？"

# 处理输入
inputs = processor(text=question, images=image, return_tensors="pt")

# 生成回答
with torch.no_grad():
    outputs = model.generate(
        **inputs,
        max_new_tokens=20,
        temperature=0.7,
        do_sample=True
    )

# 解码结果
answer = processor.decode(outputs[0], skip_special_tokens=True)
print(f"问题: {question}")
print(f"回答: {answer}")

import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import os
import glob

# 加载处理器和模型
processor = AutoProcessor.from_pretrained("smol-vlm/smolvlm-700m")
model = AutoModelForCausalLM.from_pretrained("smol-vlm/smolvlm-700m")

# 设置为评估模式
model.eval()

# 设置批处理大小
batch_size = 8

# 获取图像文件列表
image_dir = "./images"
image_files = glob.glob(os.path.join(image_dir, "*.jpg")) + \
              glob.glob(os.path.join(image_dir, "*.png"))

# 准备提示
prompt = "描述这张图片"

# 批量处理图像
for i in range(0, len(image_files), batch_size):
    # 获取当前批次的图像
    batch_files = image_files[i:i+batch_size]
    
    # 加载和处理图像
    images = []
    valid_files = []
    
    for file in batch_files:
        try:
            image = Image.open(file)
            images.append(image)
            valid_files.append(file)
        except Exception as e:
            print(f"无法加载图像 {file}: {e}")
            continue
    
    if not images:
        continue
    
    # 处理输入
    inputs = processor(text=[prompt]*len(images), images=images, return_tensors="pt", padding=True)
    
    # 生成描述
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=100,
            temperature=0.7,
            do_sample=True
        )
    
    # 解码结果
    for j, output in enumerate(outputs):
        description = processor.decode(output, skip_special_tokens=True)
        print(f"文件: {valid_files[j]}")
        print(f"描述: {description}")
        print()