Batch size reduces accuracy of ensemble of pretrained CNNs
I'm trying to implement basic softmax-based voting, which I take a couple of pretrained CNNs, softmax their outputs, add them together and then use argmax as final output.
So I loaded 4 different pretrained CNNs (vgg11, vgg13, vgg16, vgg19) from &quot;chenyaofo/pytorch-cifar-models&quot; that were trained on CIFAR10 -- I didn't train them.

When I iterate over the test set with DataLoader with batch_size=128/256,  I get to 94% accuracy;

When I iterate over the test set with batch_size=1, I get to 69% accuracy.


How could it be?
This is the code:
import torch
from tqdm import tqdm
from torchvision import datasets, transforms, models
from torch.utils.data import DataLoader
import torch.nn as nn
import torch

torch.cuda.empty_cache()

model_names = [
        &quot;cifar10_vgg11_bn&quot;,
        &quot;cifar10_vgg13_bn&quot;,
        &quot;cifar10_vgg16_bn&quot;,
        &quot;cifar10_vgg19_bn&quot;,
        # &quot;cifar10_resnet56&quot;,
]

batch_size = 2

test_transform = transforms.Compose([
                    transforms.ToTensor(),
])

def load_models():
    models = []
    for model_name in model_names:
        model = torch.hub.load(&quot;chenyaofo/pytorch-cifar-models&quot;, model_name, pretrained=True)
        models.append(model)
    return models

testset = datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=test_transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,
                                         shuffle=False)

import torch.nn as nn
import torch

class MyEnsemble(nn.Module):

    def __init__(self, modelA, modelB, modelC, modelD):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        self.modelC = modelC
        self.modelD = modelD
        # self.modelE = modelE

    def forward(self, x):
        out1 = self.modelA(x)
        out2 = self.modelB(x)
        out3 = self.modelC(x)
        out4 = self.modelD(x)
        # out5 = self.modelE(x)

        # print(out1.shape)

        out1 = torch.softmax(out1, dim=1)
        out2 = torch.softmax(out2, dim=1)
        out3 = torch.softmax(out3, dim=1)
        out4 = torch.softmax(out4, dim=1)

        out = out1 + out2 + out3 + out4

        return out

from EnsembleModule import MyEnsemble
from data import load_models, testloader
import torch
from tqdm import tqdm

device = 'cuda' if torch.cuda.is_available() else 'cpu'

models = load_models()

model = MyEnsemble(models[0], models[1], models[2], models[3])

model.to(device)

total = 0
correct = 0
with torch.no_grad():
    for images, labels in tqdm(testloader):
        images, labels = images.to(device), labels.to(device)
        outputs = model(images)
        _, predictions = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predictions == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))




You're forgetting to call model.eval():
# ...

model.to(device)
model.eval() # &lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;&lt;

total = 0
correct = 0
with torch.no_grad():
    for images, labels in tqdm(testloader):
        images, labels = images.to(device), labels.to(device)
        outputs = model(images)

# ...

As your model has BatchNorm layers, batch_size=1 is particularly degrading.
The pre-processing should also follow the one used for training. As you can see in the repository of the author of the model, you should normalize using the following statistics:
test_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))
])

