Modelo Híbrido Multimodal

import torch
import torch.nn as nn
from transformers import ViTModel, BertModel

class MultimodalModel(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        
        # Vision Transformer
        self.vision_model = ViTModel.from_pretrained(
            'google/vit-base-patch16-224'
        )
        
        # BERT para texto
        self.text_model = BertModel.from_pretrained(
            'bert-base-multilingual-cased'
        )
        
        # Capas de fusión
        self.vision_proj = nn.Linear(768, 512)
        self.text_proj = nn.Linear(768, 512)
        
        # Atención cruzada
        self.cross_attention = nn.MultiheadAttention(
            embed_dim=512, num_heads=8
        )
        
        # Clasificador final
        self.classifier = nn.Sequential(
            nn.Linear(1024, 256),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(256, num_classes)
        )
    
    def forward(self, images, texts, attention_mask):
        # Procesar imágenes
        vision_outputs = self.vision_model(images)
        vision_features = self.vision_proj(
            vision_outputs.last_hidden_state[:, 0]
        )
        
        # Procesar texto
        text_outputs = self.text_model(
            texts, attention_mask=attention_mask
        )
        text_features = self.text_proj(
            text_outputs.last_hidden_state[:, 0]
        )
        
        # Fusión con atención cruzada
        fused_features, _ = self.cross_attention(
            vision_features.unsqueeze(0),
            text_features.unsqueeze(0),
            text_features.unsqueeze(0)
        )
        
        # Concatenar características
        combined = torch.cat([
            vision_features, 
            fused_features.squeeze(0)
        ], dim=1)
        
        # Clasificación
        output = self.classifier(combined)
        return output

import torch.optim as optim
from torch.utils.data import DataLoader

# Configuración del entrenamiento
model = MultimodalModel(num_classes=10)
optimizer = optim.AdamW(model.parameters(), lr=1e-4)
criterion = nn.CrossEntropyLoss()
scheduler = optim.lr_scheduler.CosineAnnealingLR(
    optimizer, T_max=100
)

# Loop de entrenamiento
def train_epoch(model, dataloader, optimizer, criterion):
    model.train()
    total_loss = 0
    correct = 0
    total = 0
    
    for batch in dataloader:
        images, texts, masks, labels = batch
        
        # Forward pass
        outputs = model(images, texts, masks)
        loss = criterion(outputs, labels)
        
        # Backward pass
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        # Métricas
        total_loss += loss.item()
        _, predicted = outputs.max(1)
        total += labels.size(0)
        correct += predicted.eq(labels).sum().item()
    
    accuracy = 100. * correct / total
    avg_loss = total_loss / len(dataloader)
    
    return accuracy, avg_loss

# Entrenamiento completo
for epoch in range(100):
    train_acc, train_loss = train_epoch(
        model, train_loader, optimizer, criterion
    )
    
    if epoch % 10 == 0:
        print(f'Epoch {epoch}: Acc={train_acc:.2f}%, Loss={train_loss:.4f}')
    
    scheduler.step()