Why is my loss not decreasing over training 10 epochs?
My hardware is a Ryzen 5000 series cpu with an nvidia rtx 3060 gpu. I'm currently working on a school assignment involving using a deep learning model (implemented in PyTorch) to predict COVID diagnosis from CT slice images. The dataset can be found at this url on GitHub: https://github.com/UCSD-AI4H/COVID-CT
I've written a custom dataset that takes the images from the dataset and resizes them to 224x224. I've also converted all rgba or grayscale images to rgb using skimage.color. Other transforms include random horizontal and vertical flipping, as well as ToTensor(). To evaluate the model I've used sklearn.metrics to compute the AUC, F1 score, and accuracy of the model.
My trouble is that I can't get the model to train. After 10 epochs the loss has not decreased. I've tried adjusting the learning rate of my optimizer but it hasn't helped. Any recommendations/thoughts would be greatly appreciated. Thanks!
class RONANet(nn.Module):
  def __init__(self, classifier_type=None):
    super(RONANet, self).__init__()
    self.classifier_type = classifier_type
    self.relu = nn.ReLU()
    self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2)
    self.classifier = self.compose_classifier()
    self.conv_layers = nn.Sequential(
        nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1),
        nn.BatchNorm2d(32),
        self.relu,
        self.maxpool,
        nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
        nn.BatchNorm2d(64),
        self.relu,
        self.maxpool,
        nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
        nn.BatchNorm2d(128),
        self.relu,
        self.maxpool,
        nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
        self.relu,
        self.maxpool,
        nn.AdaptiveAvgPool2d(output_size=(1,1)),
    )
  def compose_classifier(self):
    if 'fc' in self.classifier_type:
            classifier = nn.Sequential(
                nn.Flatten(),
                nn.Linear(14**2*256, 256),
                self.relu,
                nn.Linear(256, 128),
                self.relu,
                nn.Linear(128, 2))
            
    elif 'conv'in self.classifier_type:
        classifier = nn.Sequential(
            nn.Conv2d(256, 1, kernel_size=1, stride=1))
        return classifier
    
  def forward(self, x):
    features = self.conv_layers(x)
    out = self.classifier(features)
    if 'conv' in self.classifier_type:
        out = out.reshape([-1,])
    return out

    
RONANetv1 = RONANet(classifier_type='conv')
RONANetv1 = RONANetv1.cuda()
RONANetv2 = RONANet(classifier_type='fc')
RONANetv2 = RONANetv2.cuda()

criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.Adam(RONANetv1.parameters(), lr=0.1)
num_epochs = 100
best_auc = 0.5 # set threshold to random model performance
scores = {}

for epoch in range(num_epochs):
RONANetv1.train()
print(f'Current Epoch: {epoch+1}')
epoch_loss = 0

for images, labels in train_dataloader:
    batch_loss = 0
    optimizer.zero_grad()
    with torch.set_grad_enabled(True):
        images = images.cuda()
        labels = labels.cuda()
        out = RONANetv1(images)
        loss = criterion(out, labels)
        batch_loss += loss.item()
        loss.backward()
        optimizer.step()
        epoch_loss += batch_loss
print(f'Loss this epoch: {epoch_loss}\n')
        
    
current_val_auc, current_val_f1, current_val_acc = get_scores(RONANetv1, val_dataloader)
if current_val_auc &gt; best_auc:
    best_auc = current_val_auc
    torch.save(RONANetv1.state_dict(), 'RONANetv1.pth')
    scores['AUC'] = current_val_auc
    scores['f1'] = current_val_f1
    scores['Accuracy'] = current_val_acc
    print(scores)

.
Output:

Current Epoch: 1
Loss this epoch: 38.038745045661926

{'AUC': 0.6632183908045978, 'f1': 0.0, 'Accuracy': 0.4915254237288136}

Current Epoch: 2
Loss this epoch: 37.96312761306763

Current Epoch: 3
Loss this epoch: 37.93656861782074

Current Epoch: 4
Loss this epoch: 38.045261442661285

Current Epoch: 5
Loss this epoch: 38.01626980304718

Current Epoch: 6
Loss this epoch: 37.93017905950546

Current Epoch: 7
Loss this epoch: 37.913547694683075

Current Epoch: 8
Loss this epoch: 38.049841582775116

Current Epoch: 9
Loss this epoch: 37.95650988817215


So the issue is you're only training the first part of the classifier and not the second
# this
optimizer = torch.optim.Adam(RONANetv1.parameters(), lr=0.1)
# needs to become this
from itertools import chain
optimizer = torch.optim.Adam(chain(RONANetv1.parameters(), RONANetv2.parameters()))

and you need to incorportate the other cnn in training too
intermediate_out = RONANetv1(images)
out = RONANetv2(intermediate_out)
loss = criterion(out, labels)
batch_loss += loss.item()
loss.backward()
optimizer.step()

Hope that helps best of luck!
