基于大模型的多模态数据融合实战应用

pip install transformers torch torchvision

import torch
from transformers import BlipProcessor, BlipForConditionalGeneration
from PIL import Image

# 加载 BLIP-2 处理器和模型
processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")

# 读取图像
image_path = "example.jpg"  # 请替换为本地图片路径
image = Image.open(image_path).convert("RGB")

# 预处理输入
inputs = processor(images=image, return_tensors="pt")

# 生成图像描述
with torch.no_grad():
    output = model.generate(**inputs)

# 解码输出
caption = processor.decode(output[0], skip_special_tokens=True)
print("生成的图像描述:", caption)

pip install openai-whisper

import whisper

# 加载 Whisper 模型
model = whisper.load_model("base")

# 处理音频文件
result = model.transcribe("speech.mp3")
text = result["text"]

print("语音转文本结果:", text)

# 将文本输入 GPT-4V（假设已接入 OpenAI API）
import openai

openai.api_key = "your_api_key"

response = openai.ChatCompletion.create(
    model="gpt-4-vision-preview",
    messages=[{"role": "user", "content": f"请根据以下语音文本总结内容:\n{text}"}]
)

print("GPT-4V 处理结果:", response["choices"][0]["message"]["content"])

import torch
import torch.nn as nn
import torchvision.models as models

class MultiModalTransformer(nn.Module):
    def __init__(self, text_dim=768, image_dim=2048, hidden_dim=512):
        super(MultiModalTransformer, self).__init__()

        # 图像编码器（使用 ResNet 提取特征）
        self.image_encoder = models.resnet50(pretrained=True)
        self.image_encoder = nn.Sequential(*list(self.image_encoder.children())[:-1])  # 去掉全连接层
        self.img_fc = nn.Linear(2048, hidden_dim)

        # 文本编码器（简单 MLP 进行降维）
        self.text_fc = nn.Linear(text_dim, hidden_dim)

        # Transformer 融合层
        self.transformer = nn.Transformer(d_model=hidden_dim, num_encoder_layers=3, num_decoder_layers=3)

    def forward(self, image, text_embedding):
        # 提取图像特征
        image_features = self.image_encoder(image)
        image_features = image_features.view(image_features.size(0), -1)
        image_features = self.img_fc(image_features)

        # 处理文本特征
        text_features = self.text_fc(text_embedding)

        # 进行 Transformer 融合
        fused_features = self.transformer(image_features.unsqueeze(1), text_features.unsqueeze(1))

        return fused_features

# 测试
image_input = torch.randn(2, 3, 224, 224)  # 假设有2张图片
text_input = torch.randn(2, 768)  # 假设文本编码维度为768
model = MultiModalTransformer()
output = model(image_input, text_input)

print("融合输出的形状:", output.shape)

class SharedAttention(nn.Module):
    def __init__(self, embed_dim):
        super(SharedAttention, self).__init__()
        self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads=8)

    def forward(self, text_features, image_features):
        # 共享注意力计算
        combined_features = torch.cat([text_features, image_features], dim=1)
        attn_output, _ = self.multihead_attn(combined_features, combined_features, combined_features)
        return attn_output

# 示例
text_features = torch.randn(4, 10, 512)  # 4个样本，每个样本10个文本 token，512 维度
image_features = torch.randn(4, 10, 512)  # 4个样本，每个样本10个图像区域，512 维度
shared_attn = SharedAttention(embed_dim=512)
output = shared_attn(text_features, image_features)

print("共享注意力融合后的形状:", output.shape)

import torch.nn.functional as F

def contrastive_loss(image_features, text_features, temperature=0.07):
    # 计算余弦相似度
    similarity = F.cosine_similarity(image_features, text_features)
    # 计算对比损失
    loss = -torch.mean(F.log_softmax(similarity / temperature, dim=-1))
    return loss

import torch.nn as nn

class DistillationLoss(nn.Module):
    def __init__(self, temperature=2.0):
        super(DistillationLoss, self).__init__()
        self.temperature = temperature

    def forward(self, teacher_logits, student_logits):
        # 计算蒸馏损失
        loss = nn.KLDivLoss()(F.log_softmax(student_logits / self.temperature, dim=-1),
                              F.softmax(teacher_logits / self.temperature, dim=-1))
        return loss