title: Pytorch based Resnet18 achieves low accuracy on CIFAR100
body: <p>I'm training a resnet18 on CIFAR100 dataset. After about 50 iterations the validation accuracy converged at about 34%. While the training accuracy reached almost 100%.</p>
  <p>I doubt it's kinda overfitting, so i applied data augmentation like <code>RandomHorizontalFlip</code> and <code>RandomRotation</code>, which made the validation converge at about 40%.</p>
  <p>I also tried decaying learning rate <code>[0.1, 0.03, 0.01, 0.003, 0.001]</code>, decaying after each 50 iterations. Decaying learning rate seems not enhancing the performance.</p>
  <p>Heard that Resnet on CIFAR100 may get 70%~80% accuracy. What trick else could I apply? Or is there anything wrong in my implementation? The same code on CIFAR10 can achieve about 80% accuracy.</p>
  <p>My whole training and evaluation code is here below:</p>
  <pre><code>import torch
  from torch import nn
  from torch import optim
  from torch.utils.data import DataLoader
  from torchvision.models import resnet18
  from torchvision.transforms import Compose, ToTensor, RandomHorizontalFlip, RandomRotation, Normalize
  from torchvision.datasets import CIFAR10, CIFAR100
  import os
  from datetime import datetime
  import matplotlib.pyplot as plt
  
  
  def draw_loss_curve(histories, legends, save_dir):
      os.makedirs(save_dir, exist_ok=True)
      for key in histories[0][0].keys():
          if key != &quot;epoch&quot;:
              plt.figure()
              plt.title(key)
              for history in histories:
                  x = [h[&quot;epoch&quot;] for h in history]
                  y = [h[key] for h in history]
                  # plt.ylim(ymin=0, ymax=3.0)
                  plt.plot(x, y)
              plt.legend(legends)
              plt.savefig(os.path.join(save_dir, key + &quot;.png&quot;))
  
  
  def cal_acc(out, label):
      batch_size = label.shape[0]
      pred = torch.argmax(out, dim=1)
      num_true = torch.nonzero(pred == label).shape[0]
      acc = num_true / batch_size
      return torch.tensor(acc)
  
  
  class LrManager(optim.lr_scheduler.LambdaLR):
      def __init__(self, optimizer, lrs):
          def f(epoch):
              rate = 1
              for k in sorted(lrs.keys()):
                  if epoch &gt;= k:
                      rate = lrs[k]
                  else:
                      break
              return rate
          super(LrManager, self).__init__(optimizer, f)
  
  
  def main(cifar=100, epochs=250, batches_show=100):
      if torch.cuda.is_available():
          device = &quot;cuda&quot;
      else:
          device = &quot;cpu&quot;
          print(&quot;warning: CUDA is not available, using CPU instead&quot;)
      dataset_cls = CIFAR10 if cifar == 10 else CIFAR100
      dataset_train = dataset_cls(root=f&quot;data/{dataset_cls.__name__}/&quot;, download=True, train=True,
                                  transform=Compose([RandomHorizontalFlip(), RandomRotation(15), ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
      dataset_val = dataset_cls(root=f&quot;data/{dataset_cls.__name__}/&quot;, download=True, train=False,
                                transform=Compose([ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
      loader_train = DataLoader(dataset_train, batch_size=128, shuffle=True)
      loader_val = DataLoader(dataset_val, batch_size=128, shuffle=True)
      model = resnet18(pretrained=False, num_classes=cifar).to(device)
      optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
      lr_scheduler = LrManager(optimizer, {0: 1.0, 50: 0.3, 100: 0.1, 150: 0.03, 200: 0.01})
      criterion = nn.CrossEntropyLoss()
  
      history = []
      model.train()
      for epoch in range(epochs):
          print(&quot;-------------------  TRAINING  -------------------&quot;)
          loss_train = 0.0
          running_loss = 0.0
          acc_train = 0.0
          running_acc = 0.0
          for batch, data in enumerate(loader_train, 1):
              img, label = data[0].to(device), data[1].to(device)
              optimizer.zero_grad()
              pred = model(img)
              loss = criterion(pred, label)
              loss.backward()
              optimizer.step()
  
              running_loss += loss.item()
              loss_train += loss.item()
              acc = cal_acc(pred, label)
              running_acc += acc.item()
              acc_train += acc.item()
  
              if batch % batches_show == 0:
                  print(f&quot;epoch: {epoch}, batch: {batch}, loss: {running_loss/batches_show:.4f}, acc: {running_acc/batches_show:.4f}&quot;)
                  running_loss = 0.0
                  running_acc = 0.0
          loss_train = loss_train / batch
          acc_train = acc_train / batch
          lr_scheduler.step()
  
          print(&quot;------------------- EVALUATING -------------------&quot;)
          with torch.no_grad():
              running_acc = 0.0
              for batch, data in enumerate(loader_val, 1):
                  img, label = data[0].to(device), data[1].to(device)
                  pred = model(img)
                  acc = cal_acc(pred, label)
                  running_acc += acc.item()
              acc_val = running_acc / batch
              print(f&quot;epoch: {epoch}, acc_val: {acc_val:.4f}&quot;)
  
          history.append({&quot;epoch&quot;: epoch, &quot;loss_train&quot;: loss_train, &quot;acc_train&quot;: acc_train, &quot;acc_val&quot;: acc_val})
      draw_loss_curve([history], legends=[f&quot;resnet18-CIFAR{cifar}&quot;], save_dir=f&quot;history/resnet18-CIFAR{cifar}[{datetime.now()}]&quot;)
  
  
  if __name__ == '__main__':
      main()
  </code></pre>