title: Image classification, network not learning anything
body: <p>So i am deshuffleing images, where they have each 4 corners at a random possition.</p>
  <pre><code>EPOCHS = 5
  activation_function = nn.LeakyReLU()
  
  LossList, AccuracyList, tllist,trlist,bllist,brlist,= [], [], [], [], [], []
  
  model = MyModel(1, activation_function).to(&quot;cuda&quot;)
  
  optimizer = torch.optim.Adam(model.parameters())
  loss_objective = nn.MSELoss()
  
  for epoch in range(EPOCHS):
      
      model.train()
  
      for x, y in training_generator:
          optimizer.zero_grad()
          predictions = model(x)
          loss = loss_objective(predictions.to(&quot;cuda&quot;),y.type(torch.FloatTensor).to(&quot;cuda&quot;))
          loss.backward()
          LossList.append(loss.item())
          optimizer.step()
      with torch.no_grad():
          
          model.eval()
          tl,tr,bl,br,acc = model.evaluate(test_generator)
          AccuracyList.append(acc)
          tllist.append(tl)
          trlist.append(tr)
          bllist.append(bl)
          brlist.append(br)
  
  
      if epoch % 10 == 0 and epoch != 0: clear_output()
  
      print(
      f'Epoch {epoch+1}, \n '
      f'Loss: {LossList[-1]}, \n '
      f'Full Accuracy: {AccuracyList[-1]}, \n '
      f'Top left Accuracy: {tllist[-1]}, \n '
      f'Top right Accuracy: {trlist[-1]}, \n '
      f'Bottom left Accuracy: {bllist[-1]}, \n '
      f'Bottom right Accuracy: {brlist[-1]}'
      
      )
      print(&quot;-&quot;*50)
  </code></pre>
  <p>The problem i have is that the model(VGG-11 with Batch Normalization after the convolutional layers) does not learn anything, and the accuracy stays at around 4% for correct full image, and 25% for each corner. The size of predictions is (batch_size, 4) and for labels its the same, with inegers between 0-3.</p>
  <p>I think the problem is with how i'm suppose to make the model learn anything. Here is the model:</p>
  <pre><code># @title Define model
  
  class MyModel(nn.Module):
    def __init__(self, n_c, ActFunc):
      super(MyModel, self).__init__()
  
  
      self.layer1 = nn.Sequential(
          nn.Conv2d(3, 64, 3, padding=1),
          nn.BatchNorm2d(64),
          ActFunc,
          nn.MaxPool2d(2, 2)
      )
  
      self.layer2 = nn.Sequential(
          nn.Conv2d(64, 128, 3, padding=1),
          nn.BatchNorm2d(128),
          ActFunc,
          nn.MaxPool2d(2, 2)
      )
  
      self.layer3 = nn.Sequential(
          nn.Conv2d(128, 256, 3, padding=1),
          nn.BatchNorm2d(256),
          ActFunc
      )
  
      self.layer4 = nn.Sequential(
          nn.Conv2d(256, 256, 3, padding=1),
          nn.BatchNorm2d(256),
          ActFunc,
          nn.MaxPool2d(2, 2)
      )
  
      self.layer5 = nn.Sequential(
          nn.Conv2d(256, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc
      )
  
      self.layer6 = nn.Sequential(
          nn.Conv2d(512, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc,
          nn.MaxPool2d(2, 2)
      )
  
      self.layer7 = nn.Sequential(
          nn.Conv2d(512, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc
      )
  
      self.layer8 = nn.Sequential(
          nn.Conv2d(512, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc,
          nn.MaxPool2d(2, 2)
      )
  
      self.layer9 = nn.Sequential(
          nn.Conv2d(512, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc
      )
  
      self.layer10 = nn.Sequential(
          nn.Conv2d(512, 512, 3, padding=1),
          nn.BatchNorm2d(512),
          ActFunc,
          nn.MaxPool2d(2, 2),
          nn.Flatten()
      )
  
      self.layer11 = nn.Sequential(
          nn.Linear(512, 4096),
          ActFunc
      )
  
      self.layer12 = nn.Sequential(
          nn.Linear(4096, 4096),
          ActFunc
      )
  
      self.layer12 = nn.Sequential(
          nn.Linear(4096, 4),
          ActFunc
      )
  
  
  
    def forward(self, x):
      l1 = self.layer1(x)
      l2 = self.layer2(l1)
      l3 = self.layer3(l2)
      l4 = self.layer4(l3)
      l5 = self.layer5(l4)
      l6 = self.layer6(l5)
      l7 = self.layer7(l6)
      l8 = self.layer8(l7)
      l9 = self.layer9(l8)
      l10 = self.layer10(l9)
      l11 = self.layer11(l10)
      l12 = self.layer12(l11)
      out = torch.argsort(l12).type(torch.FloatTensor).requires_grad_()
        
  
      return out
  
    def evaluate(self, test_generator):
        batchfullacc = 0
        batchacc = 0
        for images, labels in test_generator:
            predictions = self.forward(images)
            batchacc += (predictions.to(&quot;cuda&quot;) == labels).sum(dim=0)
            batchfullacc += (predictions.to(&quot;cuda&quot;) == labels).all(dim=1).sum()
        tl,tr,bl,br = batchacc/len(x_te)
        acc = batchfullacc/len(x_te)
  
        return tl,tr,bl,br,acc
  </code></pre>
  <p>The training set is on 2500 images. The output i get is(this is only on 5 epochs, but nothing changes with more epochs):</p>
  <pre><code>Epoch 5, 
   Loss: 2.5899999141693115, 
   Full Accuracy: 0.042250003665685654, 
   Top left Accuracy: 0.2522500157356262, 
   Top right Accuracy: 0.2407500147819519, 
   Bottom left Accuracy: 0.25475001335144043, 
   Bottom right Accuracy: 0.25600001215934753
  --------------------------------------------------
  </code></pre>