Pytorch based Resnet18 achieves low accuracy on CIFAR100
I'm training a resnet18 on CIFAR100 dataset. After about 50 iterations the validation accuracy converged at about 34%. While the training accuracy reached almost 100%.
I doubt it's kinda overfitting, so i applied data augmentation like RandomHorizontalFlip and RandomRotation, which made the validation converge at about 40%.
I also tried decaying learning rate [0.1, 0.03, 0.01, 0.003, 0.001], decaying after each 50 iterations. Decaying learning rate seems not enhancing the performance.
Heard that Resnet on CIFAR100 may get 70%~80% accuracy. What trick else could I apply? Or is there anything wrong in my implementation? The same code on CIFAR10 can achieve about 80% accuracy.
My whole training and evaluation code is here below:
import torch
from torch import nn
from torch import optim
from torch.utils.data import DataLoader
from torchvision.models import resnet18
from torchvision.transforms import Compose, ToTensor, RandomHorizontalFlip, RandomRotation, Normalize
from torchvision.datasets import CIFAR10, CIFAR100
import os
from datetime import datetime
import matplotlib.pyplot as plt


def draw_loss_curve(histories, legends, save_dir):
    os.makedirs(save_dir, exist_ok=True)
    for key in histories[0][0].keys():
        if key != &quot;epoch&quot;:
            plt.figure()
            plt.title(key)
            for history in histories:
                x = [h[&quot;epoch&quot;] for h in history]
                y = [h[key] for h in history]
                # plt.ylim(ymin=0, ymax=3.0)
                plt.plot(x, y)
            plt.legend(legends)
            plt.savefig(os.path.join(save_dir, key + &quot;.png&quot;))


def cal_acc(out, label):
    batch_size = label.shape[0]
    pred = torch.argmax(out, dim=1)
    num_true = torch.nonzero(pred == label).shape[0]
    acc = num_true / batch_size
    return torch.tensor(acc)


class LrManager(optim.lr_scheduler.LambdaLR):
    def __init__(self, optimizer, lrs):
        def f(epoch):
            rate = 1
            for k in sorted(lrs.keys()):
                if epoch &gt;= k:
                    rate = lrs[k]
                else:
                    break
            return rate
        super(LrManager, self).__init__(optimizer, f)


def main(cifar=100, epochs=250, batches_show=100):
    if torch.cuda.is_available():
        device = &quot;cuda&quot;
    else:
        device = &quot;cpu&quot;
        print(&quot;warning: CUDA is not available, using CPU instead&quot;)
    dataset_cls = CIFAR10 if cifar == 10 else CIFAR100
    dataset_train = dataset_cls(root=f&quot;data/{dataset_cls.__name__}/&quot;, download=True, train=True,
                                transform=Compose([RandomHorizontalFlip(), RandomRotation(15), ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
    dataset_val = dataset_cls(root=f&quot;data/{dataset_cls.__name__}/&quot;, download=True, train=False,
                              transform=Compose([ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
    loader_train = DataLoader(dataset_train, batch_size=128, shuffle=True)
    loader_val = DataLoader(dataset_val, batch_size=128, shuffle=True)
    model = resnet18(pretrained=False, num_classes=cifar).to(device)
    optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
    lr_scheduler = LrManager(optimizer, {0: 1.0, 50: 0.3, 100: 0.1, 150: 0.03, 200: 0.01})
    criterion = nn.CrossEntropyLoss()

    history = []
    model.train()
    for epoch in range(epochs):
        print(&quot;-------------------  TRAINING  -------------------&quot;)
        loss_train = 0.0
        running_loss = 0.0
        acc_train = 0.0
        running_acc = 0.0
        for batch, data in enumerate(loader_train, 1):
            img, label = data[0].to(device), data[1].to(device)
            optimizer.zero_grad()
            pred = model(img)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()
            loss_train += loss.item()
            acc = cal_acc(pred, label)
            running_acc += acc.item()
            acc_train += acc.item()

            if batch % batches_show == 0:
                print(f&quot;epoch: {epoch}, batch: {batch}, loss: {running_loss/batches_show:.4f}, acc: {running_acc/batches_show:.4f}&quot;)
                running_loss = 0.0
                running_acc = 0.0
        loss_train = loss_train / batch
        acc_train = acc_train / batch
        lr_scheduler.step()

        print(&quot;------------------- EVALUATING -------------------&quot;)
        with torch.no_grad():
            running_acc = 0.0
            for batch, data in enumerate(loader_val, 1):
                img, label = data[0].to(device), data[1].to(device)
                pred = model(img)
                acc = cal_acc(pred, label)
                running_acc += acc.item()
            acc_val = running_acc / batch
            print(f&quot;epoch: {epoch}, acc_val: {acc_val:.4f}&quot;)

        history.append({&quot;epoch&quot;: epoch, &quot;loss_train&quot;: loss_train, &quot;acc_train&quot;: acc_train, &quot;acc_val&quot;: acc_val})
    draw_loss_curve([history], legends=[f&quot;resnet18-CIFAR{cifar}&quot;], save_dir=f&quot;history/resnet18-CIFAR{cifar}[{datetime.now()}]&quot;)


if __name__ == '__main__':
    main()


Resnet18 from torchvision.models it's an ImageNet implementation. Because ImageNet samples much bigger(224x224) than CIFAR10/100 (32x32), the first layers designed to aggressively downsample the input ('stem Network'). It's lead to missing much valuable information on small CIFAR10/100 images.
To achieve good accuracy on CIFAR10, authors use different network structure as described in original paper:
https://arxiv.org/pdf/1512.03385.pdf
and explained in this article:
https://towardsdatascience.com/resnets-for-cifar-10-e63e900524e0
You can download resnet fo CIFAR10 from this repo: https://github.com/akamaster/pytorch_resnet_cifar10
