pytorch 1D Dropout leads to unstable learning
I'm implementing an Inception-like CNN in pytorch.  After the blocks of convolution layers, I have three fully-connected linear layers followed by a sigmoid activation to give me my final regression output.  I'm testing the effects of dropout layers in this network, but it's giving me some unexpected results.

Here is the code:

class MyInception(nn.Module):
    def __init__(self, in_channels, verbose=False):
        super(MyInception, self).__init__()
        self.v = verbose
        ic=in_channels; oc=16
        self.inceptionBlock1 = InceptionBlock(in_channels=ic, out_channels=oc, maxpool=False, verbose=verbose) 
        self.inceptionBlock2 = InceptionBlock(in_channels=oc * 6, out_channels=oc, maxpool=False, verbose=verbose) 
        self.pool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        self.regressor = nn.Sequential(
            nn.Linear(oc * 6 * 35 * 35, 1024, bias=True),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.2, inplace=False),  # &lt;--- Dropout 1
            nn.Linear(1024, 128, bias=True),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.2, inplace=False),  # &lt;--- Dropout 2
            nn.Linear(128, 1, bias=True),
            nn.Sigmoid()
        )

    def forward(self, x):
        x = self.inceptionBlock1(x)
        x = self.inceptionBlock2(x)
        x = self.pool(x)
        x = torch.flatten(x, 1)
        x = self.regressor(x)
        return x


def train(epochs=10, dot_every=25):
    running = pd.DataFrame(columns=['Epoch','Round','TrainLoss','TestLoss','LearningRate'])
    for epoch in range(epochs):
        train_losses = []
        model.train()
        counter = 0

        for images, targets in train_loader:
            images = images.to(device)
            targets = targets.to(device)

            optimizer.zero_grad()
            outputs = model(images)
            loss = loss_fn(torch.flatten(outputs), targets)
            train_losses.append( loss.item() )
            loss.backward()
            optimizer.step()

            counter += 1
            if counter % dot_every == 0: 
                print(".",  end='.', flush=True)
                test_loss = test()
            else:
                test_loss = -1.
            lr = np.squeeze(scheduler.get_lr())
            running = running.append(pd.Series([epoch, counter, loss.item(), test_loss, lr], index=running.columns), ignore_index=True)

        test_loss = test()
        train_loss = np.mean(np.asarray(train_losses))
        running = running.append(pd.Series([epoch, counter, train_loss, test_loss, lr], index=running.columns), ignore_index=True)
        print("")
        print(f"Epoch {epoch+1}, Train Loss: {np.round(train_loss,4)}, Test Loss: {np.round(test_loss, 4)}, Learning Rate: {np.format_float_scientific(lr, precision=4)}")
    return running


def test():
    model.eval()
    test_losses = []
    for i, (images,targets) in enumerate(test_loader):
        images = images.to(device)
        targets = targets.to(device)
        outputs = model(images)
        loss = loss_fn(torch.flatten(outputs), targets)
        test_losses.append( loss.item() )

    mean_loss = np.mean(np.asarray(test_losses))
    return mean_loss

# instantiate the model
model = MyInception(in_channels=4, verbose=False).to(device)
# define the optimizer and loss function
optimizer = Adam(model.parameters(), lr=0.001, weight_decay=0.0001)
loss_fn = nn.MSELoss()

# run it
results = train(epochs=10, dot_every=20)



Here is a plot of the MSE losses for the training data.  (red = no dropout, green = second dropout only, blue = first dropout only, purple = both dropouts)
Runs with dropout have big increases in losses at the epoch boundaries (dashed vertical lines), with the double dropout even having a big jump in loss at the start of epoch 10.



The important thing is the test loss.  That is much more stable and not too different between either condition after the 5th epoch, so maybe I shouldn't care. But I would like to understand what is going on.



I cracked the case.  I realized that I flip model.train() to model.eval() in the test call without setting it back to train() after.  Since Dropout behaves differently in train and eval modes, adding in Dropout revealed the bug.
