Transfer Learning & Pre-trained Models — AI Tutorials

Why Transfer Learning Works

A network trained on ImageNet (1.2M images, 1000 classes) learns a rich hierarchy of visual features:

Layer depth	What is learned
Early layers	Edges, corners, colour blobs
Middle layers	Textures, patterns, object parts
Late layers	Semantic concepts (fur, wheels, faces)

Because natural images share low-level statistics, these features transfer broadly — even to medical imaging, satellite imagery, and industrial inspection tasks.

Strategies

Strategy	Freeze	Update	Best for
Feature extraction	All backbone layers	Classifier head only	Very small datasets (<1K)
Partial fine-tuning	Early layers	Last N blocks + head	Medium datasets
Full fine-tuning	Nothing	Entire network	Large datasets, similar domain
Layer-wise LR decay	Nothing	All layers, decreasing LR	Large datasets, distant domain

Choosing the Right Backbone

Model	Params	ImageNet Top-1	Latency (CPU ms)
ResNet-18	11.7M	69.8%	~45
ResNet-50	25.6M	80.9%	~90
EfficientNet-B0	5.3M	77.7%	~50
EfficientNet-B4	19.3M	83.4%	~120
MobileNet-V3-S	2.5M	67.7%	~15
ConvNeXt-Tiny	28.6M	82.1%	~95

EfficientNet Compound Scaling

EfficientNet scales depth $d$ , width $w$ , and resolution $r$ jointly:

$d = \alpha^\phi, \quad w = \beta^\phi, \quad r = \gamma^\phi$

subject to $\alpha \cdot \beta^2 \cdot \gamma^2 \approx 2$ (constant FLOP budget). Scaling all three together outperforms scaling any single dimension.

Layer-wise Learning Rate Decay (LLRD)

Higher layers need larger updates while lower layers are already well-trained. LLRD assigns exponentially decreasing LRs to earlier layers:

$\text{lr}_{\text{layer } i \text{ from top}} = \text{lr}_{base} \times \text{decay}^i$

Grad-CAM: Gradient-weighted Class Activation Maps

Grad-CAM explains predictions by computing the gradient of the class score $y^c$ with respect to feature map activations $A^k$ :

$\alpha_k^c = \frac{1}{Z} \sum_i \sum_j \frac{\partial y^c}{\partial A_{ij}^k}$

The heatmap is: $L^c = \text{ReLU}(\sum_k \alpha_k^c A^k)$

Loading pre-trained backbones and fine-tuningpython

import torch
import torch.nn as nn
from torchvision import models
from torchvision.models import ResNet50_Weights, EfficientNet_B0_Weights

# Load ResNet-50 and replace head
resnet = models.resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)
resnet.fc = nn.Linear(resnet.fc.in_features, 10)  # 10 classes

# Load EfficientNet-B0 and replace head
effnet = models.efficientnet_b0(weights=EfficientNet_B0_Weights.IMAGENET1K_V1)
effnet.classifier[1] = nn.Linear(effnet.classifier[1].in_features, 10)

def count_params(model, trainable_only=True):
    if trainable_only:
        return sum(p.numel() for p in model.parameters() if p.requires_grad)
    return sum(p.numel() for p in model.parameters())

# Feature extraction: freeze backbone, train only head
def freeze_backbone(model, unfreeze_last_n=0):
    for param in model.parameters():
        param.requires_grad = False
    if unfreeze_last_n > 0 and hasattr(model, 'layer4'):
        for layer in [model.layer1, model.layer2, model.layer3, model.layer4][-unfreeze_last_n:]:
            for p in layer.parameters(): p.requires_grad = True
    for name in ['fc', 'classifier', 'head']:
        head = getattr(model, name, None)
        if head:
            for p in head.parameters(): p.requires_grad = True

freeze_backbone(resnet, unfreeze_last_n=0)
print(f"Trainable: {count_params(resnet)/1e6:.2f}M / {count_params(resnet, False)/1e6:.2f}M")

# Layer-wise LR decay optimizer
import torch.optim as optim

def get_llrd_optimizer(model, base_lr=1e-4, decay=0.65):
    groups = [{'params': model.fc.parameters(), 'lr': base_lr}]
    for i, name in enumerate(['layer4', 'layer3', 'layer2', 'layer1', 'bn1', 'conv1']):
        layer = getattr(model, name, None)
        if layer:
            groups.append({'params': layer.parameters(), 'lr': base_lr * (decay ** (i+1))})
    return optim.AdamW(groups, weight_decay=1e-4)

# Fine-tuning loop
from torch.optim.lr_scheduler import CosineAnnealingLR

def fine_tune(model, train_loader, val_loader, epochs=20, device='cuda'):
    model = model.to(device)
    criterion = nn.CrossEntropyLoss(label_smoothing=0.1)
    optimizer = get_llrd_optimizer(model)
    scheduler = CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-6)

    best_acc = 0.0
    for epoch in range(epochs):
        model.train()
        for images, labels in train_loader:
            images, labels = images.to(device), labels.to(device)
            optimizer.zero_grad()
            loss = criterion(model(images), labels)
            loss.backward()
            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
            optimizer.step()
        scheduler.step()

        model.eval()
        correct = total = 0
        with torch.no_grad():
            for images, labels in val_loader:
                preds = model(images.to(device)).argmax(dim=1)
                correct += (preds == labels.to(device)).sum().item()
                total += labels.size(0)
        acc = correct / total
        if acc > best_acc:
            best_acc = acc
            torch.save(model.state_dict(), 'best_model.pth')
        print(f"Epoch {epoch+1}  val_acc={acc:.4f}")

Grad-CAM implementationpython

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt

class GradCAM:
    def __init__(self, model, target_layer):
        self.model = model
        self.gradients = None
        self.activations = None
        target_layer.register_forward_hook(self._save_activations)
        target_layer.register_full_backward_hook(self._save_gradients)

    def _save_activations(self, module, input, output):
        self.activations = output.detach()

    def _save_gradients(self, module, grad_input, grad_output):
        self.gradients = grad_output[0].detach()

    def generate(self, input_tensor, class_idx=None):
        self.model.eval()
        output = self.model(input_tensor)
        if class_idx is None:
            class_idx = output.argmax(dim=1).item()
        self.model.zero_grad()
        output[0, class_idx].backward()

        weights = self.gradients.mean(dim=(2, 3), keepdim=True)
        cam = (weights * self.activations).sum(dim=1, keepdim=True)
        cam = F.relu(cam)
        cam = F.interpolate(cam, size=input_tensor.shape[-2:], mode='bilinear', align_corners=False)
        cam = cam.squeeze().cpu().numpy()
        cam = (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)
        return cam, class_idx

# Usage: hook into last ResNet conv layer
gradcam = GradCAM(resnet, resnet.layer4[-1].conv2)
cam, pred = gradcam.generate(image_tensor.unsqueeze(0))

Knowledge check

You have 800 labelled training images for a medical scan classification task. Which strategy is most appropriate?

Knowledge check

What does layer-wise learning rate decay (LLRD) achieve?

Knowledge check

What does Grad-CAM visualise?

← Computer Vision