SuperAPI AI Directory

deep-learning

General & other

View repository

Updated 10-16 03:43

About this skill

Deep Learning Skill

Build and train neural networks using PyTorch.

Quick Start

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
Define model

class SimpleNN(nn.Module): def init(self, input_dim, hidden_dim, output_dim): super().init() self.layers = nn.Sequential( nn.Linear(input_dim, hidden_dim), nn.ReLU(), nn.Dropout(0.3), nn.Linear(hidden_dim, output_dim) )

def forward(self, x):
    return self.layers(x)
Train

model = SimpleNN(10, 64, 2) optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3) criterion = nn.CrossEntropyLoss()

for epoch in range(10): model.train() for batch_x, batch_y in train_loader: optimizer.zero_grad() output = model(batch_x) loss = criterion(output, batch_y) loss.backward() optimizer.step()


## Key Topics

### 1. Neural Network Architectures

| Architecture | Use Case | Key Layers |
|-------------|----------|------------|
| **MLP** | Tabular data | Linear, ReLU, Dropout |
| **CNN** | Images | Conv2d, MaxPool2d, BatchNorm |
| **RNN/LSTM** | Sequences | LSTM, GRU |
| **Transformer** | NLP, Vision | MultiheadAttention |

```python
class MLP(nn.Module):
    def __init__(self, dims, dropout=0.3):
        super().__init__()
        layers = []
        for i in range(len(dims) - 1):
            layers.extend([
                nn.Linear(dims[i], dims[i+1]),
                nn.BatchNorm1d(dims[i+1]) if i < len(dims) - 2 else nn.Identity(),
                nn.ReLU() if i < len(dims) - 2 else nn.Identity(),
                nn.Dropout(dropout) if i < len(dims) - 2 else nn.Identity()
            ])
        self.network = nn.Sequential(*layers)

    def forward(self, x):
        return self.network(x)

2. Training Loop Template

import torch.cuda.amp as amp

def train_epoch(model, loader, optimizer, criterion, device, scaler=None):
    model.train()
    total_loss = 0

    for batch_x, batch_y in loader:
        batch_x, batch_y = batch_x.to(device), batch_y.to(device)

        optimizer.zero_grad()

        if scaler:  # Mixed precision
            with amp.autocast():
                output = model(batch_x)
                loss = criterion(output, batch_y)
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()
        else:
            output = model(batch_x)
            loss = criterion(output, batch_y)
            loss.backward()
            optimizer.step()

        total_loss += loss.item()

    return total_loss / len(loader)

3. Learning Rate Scheduling

from torch.optim.lr_scheduler import OneCycleLR, CosineAnnealingLR

# OneCycle (recommended)
scheduler = OneCycleLR(
    optimizer,
    max_lr=1e-3,
    epochs=epochs,
    steps_per_epoch=len(train_loader)
)

# Cosine Annealing
scheduler = CosineAnnealingLR(optimizer, T_max=epochs)

4. Regularization

Technique Implementation
Dropout nn.Dropout(p=0.3)
Weight Decay AdamW(weight_decay=0.01)
Batch Norm nn.BatchNorm1d(dim)
Early Stopping Monitor val_loss

5. Model Checkpointing

def save_checkpoint(model, optimizer, epoch, val_loss, path):
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'val_loss': val_loss
    }, path)

def load_checkpoint(model, optimizer, path):
    checkpoint = torch.load(path)
    model.load_state_dict(checkpoint['model_state_dict'])
    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
    return checkpoint['epoch'], checkpoint['val_loss']

Best Practices

DO

  • Use mixed precision training (AMP)
  • Apply gradient clipping
  • Save checkpoints regularly
  • Use AdamW over Adam
  • Monitor learning curves
  • Initialize weights properly

DON'T

  • Don't use large batches without LR scaling
  • Don't train without validation
  • Don't skip weight initialization
  • Don't ignore NaN losses

Exercises

Exercise 1: Basic MLP

# TODO: Build an MLP for MNIST classification
# Architecture: 784 -> 256 -> 128 -> 10

Exercise 2: Training with AMP

# TODO: Implement mixed precision training
# and compare speed with FP32 training

Unit Test Template

import pytest
import torch

def test_model_forward():
    """Test model forward pass."""
    model = SimpleNN(10, 64, 2)
    x = torch.randn(32, 10)

    output = model(x)

    assert output.shape == (32, 2)

def test_model_backward():
    """Test gradient flow."""
    model = SimpleNN(10, 64, 2)
    x = torch.randn(32, 10)
    y = torch.randint(0, 2, (32,))

    output = model(x)
    loss = nn.CrossEntropyLoss()(output, y)
    loss.backward()

    # Check gradients exist
    for param in model.parameters():
        assert param.grad is not None

Troubleshooting

Problem Cause Solution
NaN loss Exploding gradients Add gradient clipping
Loss plateau LR too low Increase LR or use scheduler
Overfitting Model too complex Add dropout, reduce layers
GPU OOM Batch too large Reduce batch size

Related Resources

  • Agent: 04-deep-learning
  • Previous: clustering
  • Next: nlp-basics
  • Docs: PyTorch Tutorials

Version: 1.4.0 | Status: Production Ready