原生多模态推理：Gemini 3 如何“理解世界”而非“拼接图像”？

import torch
import torch.nn as nn
import torch.optim as optim

class MultiModalFusion(nn.Module):
    def __init__(self, text_dim, image_dim, audio_dim, hidden_dim):
        super(MultiModalFusion, self).__init__()
        self.text_fc = nn.Linear(text_dim, hidden_dim)
        self.image_fc = nn.Linear(image_dim, hidden_dim)
        self.audio_fc = nn.Linear(audio_dim, hidden_dim)
        self.fusion_fc = nn.Linear(hidden_dim * 3, hidden_dim)

    def forward(self, text, image, audio):
        text_features = torch.relu(self.text_fc(text))
        image_features = torch.relu(self.image_fc(image))
        audio_features = torch.relu(self.audio_fc(audio))

        fused_features = torch.cat((text_features, image_features, audio_features), dim=1)
        fused_features = torch.relu(self.fusion_fc(fused_features))
        return fused_features

# 示例输入
text_input = torch.randn(1, 512)  # 文本特征
image_input = torch.randn(1, 1024)  # 图像特征
audio_input = torch.randn(1, 256)  # 音频特征

# 创建模型实例
model = MultiModalFusion(text_dim=512, image_dim=1024, audio_dim=256, hidden_dim=512)

# 前向传播
output = model(text_input, image_input, audio_input)
print(output.shape)  # 输出形状应为 (1, 512)

import torch
import torch.nn as nn

class CrossModalAttention(nn.Module):
    def __init__(self, embed_dim):
        super(CrossModalAttention, self).__init__()
        self.query = nn.Linear(embed_dim, embed_dim)
        self.key = nn.Linear(embed_dim, embed_dim)
        self.value = nn.Linear(embed_dim, embed_dim)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, query, key, value):
        Q = self.query(query)
        K = self.key(key)
        V = self.value(value)

        attention_scores = torch.matmul(Q, K.transpose(-2, -1)) / (embed_dim ** 0.5)
        attention_weights = self.softmax(attention_scores)
        output = torch.matmul(attention_weights, V)
        return output

# 示例输入
query_input = torch.randn(1, 10, 512)  # 查询序列
key_input = torch.randn(1, 10, 512)  # 键序列
value_input = torch.randn(1, 10, 512)  # 值序列

# 创建模型实例
model = CrossModalAttention(embed_dim=512)

# 前向传播
output = model(query_input, key_input, value_input)
print(output.shape)  # 输出形状应为 (1, 10, 512)

import torch
import torch.nn as nn
import torch.optim as optim

class MultiTaskLoss(nn.Module):
    def __init__(self):
        super(MultiTaskLoss, self).__init__()
        self.classification_loss = nn.CrossEntropyLoss()
        self.regression_loss = nn.MSELoss()

    def forward(self, outputs, targets):
        classification_output, regression_output = outputs
        classification_target, regression_target = targets
        loss = self.classification_loss(classification_output, classification_target) + \
               self.regression_loss(regression_output, regression_target)
        return loss

# 示例输入
classification_output = torch.randn(1, 10)  # 分类输出
regression_output = torch.randn(1, 1)  # 回归输出
classification_target = torch.tensor([3])  # 分类目标
regression_target = torch.tensor([0.5])  # 回归目标

# 创建损失函数实例
loss_fn = MultiTaskLoss()

# 计算损失
loss = loss_fn((classification_output, regression_output), (classification_target, regression_target))
print(loss.item())

# 假设我们已经有了用户的历史听歌记录、专辑封面和音乐片段
user_history = torch.randn(1, 512)  # 用户历史听歌记录
album_cover = torch.randn(1, 1024)  # 专辑封面
music_clip = torch.randn(1, 256)  # 音乐片段

# 创建多模态融合模型实例
fusion_model = MultiModalFusion(text_dim=512, image_dim=1024, audio_dim=256, hidden_dim=512)

# 前向传播
fused_features = fusion_model(user_history, album_cover, music_clip)

# 假设我们有一个分类器来预测用户的音乐偏好
classifier = nn.Linear(512, 10)  # 10个音乐类别
preference_prediction = classifier(fused_features)

# 打印预测结果
print(preference_prediction)