Editing Transfer Learning (section)

== <span style="color: #FFFFFF;">Applying</span> ==
'''Fine-tuning a pre-trained ResNet for a custom image classification task:'''

<syntaxhighlight lang="python">
import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torchvision.datasets import ImageFolder

# Load pre-trained ResNet-50
model = models.resnet50(weights='IMAGENET1K_V2')

# === Strategy 1: Feature Extraction (small dataset <1k images) ===
# Freeze ALL pre-trained weights
for param in model.parameters():
    param.requires_grad = False

# Replace final layer with new head for 5 classes
model.fc = nn.Linear(model.fc.in_features, 5)
# Only model.fc parameters are trainable

# === Strategy 2: Fine-tuning (larger dataset 1k+ images) ===
# Unfreeze last 2 residual blocks
for name, param in model.named_parameters():
    if 'layer4' in name or 'layer3' in name or 'fc' in name:
        param.requires_grad = True
    else:
        param.requires_grad = False

# Use different learning rates: lower for frozen/pre-trained, higher for new head
optimizer = torch.optim.Adam([
    {'params': model.layer3.parameters(), 'lr': 1e-5},   # small LR
    {'params': model.layer4.parameters(), 'lr': 1e-4},   # medium LR
    {'params': model.fc.parameters(), 'lr': 1e-3},        # large LR
])

# === Preprocessing must match pre-training ===
transform = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])  # ImageNet stats
])

dataset = ImageFolder("data/my_dataset/", transform=transform)
loader = DataLoader(dataset, batch_size=32, shuffle=True)
</syntaxhighlight>

; Strategy selection guide
: '''<100 examples per class''' → Feature extraction only; freeze all backbone layers; only train head
: '''100–1000 examples''' → Unfreeze last 1–2 blocks + head; use low LR for backbone
: '''1000–10k examples''' → Fine-tune from last half of backbone; discriminative LRs
: '''>10k examples''' → Full fine-tuning or even train from scratch if very different domain
: '''No labeled data''' → Zero-shot (CLIP, GPT) or self-supervised domain adaptation
</div>

<div style="background-color: #8B4500; color: #FFFFFF; padding: 20px; border-radius: 8px; margin-bottom: 15px;">