Equal output values given for Multiclass Classification
I'm trying to build a CNN for predicting the number of fingers in an image, using PyTorch. The network:

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.Layer1 = nn.Sequential(
        nn.Conv2d(in_channels=3, out_channels=16, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=(2, 2)),
        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=128, out_channels=256, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=(2, 2)),
        nn.Conv2d(in_channels=256, out_channels=16, kernel_size=(1, 1)),
        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=128, out_channels=256, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=(2, 2)),
        nn.Conv2d(in_channels=256, out_channels=16, kernel_size=(1, 1)),
        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=(2, 2)),
        nn.Conv2d(in_channels=128, out_channels=16, kernel_size=(1, 1)),
        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=(2, 2)),
        nn.Conv2d(in_channels=128, out_channels=16, kernel_size=(1, 1)),
        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3)),
        nn.ReLU(),
        nn.Conv2d(in_channels=128, out_channels=256, kernel_size=(3, 3)),
        nn.ReLU(),
        )

        self.Layer2 = nn.Sequential(
        nn.Linear(1536, 100),
        nn.Tanh(),
        nn.Linear(100, 6),
        nn.Softmax()
        )
        self.optimizer = optimizers.Adadelta(self.parameters())

    def forward(self, X):
        X = self.Layer1(X)
        print(X.shape)
        X = self.Layer2(X.reshape(1, 1536))
        X = X.squeeze()

        return X

    def calc_loss(self, X, num):
        out = self.forward(X).unsqueeze(dim=0)
        print("Output: "+str(out))
        target = torch.tensor([num], dtype=torch.int64).cuda()
        criterion = nn.CrossEntropyLoss()
        loss = criterion(out, target)
        return loss

    def train_step(self, X, Y):
        loss = self.calc_loss(X, Y)
        print(loss)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()


However, after the training is complete, all predictions have almost the same values (around 0.15 ~ 0.18).

It appears as though the network averages out the output probabilities to minimize loss, instead of learning the actual values.

I get the same result whether I use Softmax for the last layer with cross-entropy loss, or Sigmoid with binary cross-entropy, MSE, or SmoothL1Loss.

In case of using Adam optimizer, I get similar results, only in the range of 1e-12 ~ 1e-14.

What am I missing?

If you are using CrossEntropyLoss, you don't need to use Softmax in your forward. It is already included to CrossEntropyLoss, so you need the "raw" output. But, if you need Softmax during inference time, use NLLLoss + 'Softmax' instead.

You can find more info here
