How to do Normalization in CNN?
I am new to CNN, and I am learning it with Food Classification. Here is my code. In the DATASET part, I change the train dataset and validation dataset from numpy to tensor. At that point, the shape of tensor is ([9866, 128, 128, 3]). Since the channel 3 need to be in the first index, so I use &quot;transpose&quot; method to change the index. Then, I used the &quot;Data.TensorDataset&quot; to put training data and training label together, and the reason to use &quot;Data.DataLoader&quot; is that I need the batch size to speed up.
import os
import numpy as np
import pandas as pd
import cv2
import torch
import torch.nn as nn
from torch.nn import functional as F
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch import optim
import pandas as pd
from torch.utils.data import DataLoader, Dataset
import torch.utils.data as Data



'''Initialize Params'''
epochs = 3
learning_rate = 0.0001
momentum = 0.5
batch_size = 128


'''Load Data'''
def readFile(path,label):
    image_dir = sorted(os.listdir(path))
    # x stores photos
    x = np.zeros((len(image_dir),128,128,3),dtype=np.uint8)
    # y stores labels
    y = np.zeros((len(image_dir)), dtype=np.uint8)
    for i, file in enumerate(image_dir):
        img = cv2.imread(os.path.join(path, file))
        x[i, :, :] = cv2.resize(img,(128, 128))
        if label:
            y[i] = int(file.split(&quot;_&quot;)[0])
    if label:
        return x,y
    else:
        return x

train_x, train_y = readFile('./food/training',True)
val_x, val_y = readFile('./food/validation',True)
test_x = readFile('./food/testing',False)
# print(&quot;Reading data: &quot;)
# print(&quot;Size of training data = {}&quot;.format(len(train_x)))
# print(&quot;Size of validation data = {}&quot;.format(len(val_x)))
# print(&quot;Size of Testing data = {}&quot;.format(len(test_x)))


'''DataSet'''
train_x = torch.tensor(train_x)
# print(train_x.shape)
train_x = train_x.transpose(1,3).float()
train_y = torch.tensor(train_y)
val_x = torch.tensor(val_x)
val_x = val_x.transpose(1, 3).float()
val_y = torch.tensor(val_y)


train_dataset = Data.TensorDataset(train_x,train_y)
val_dataset = Data.TensorDataset(val_x,val_y)

train_loader = Data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
val_loader = Data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=True) 

I got the 68% accuracy of training set and I would like to improve it. I searched on net and found that maybe I should add the normalization. But I only found the way like this
transform = transforms.Compose([
    transforms.ToTensor(), # range [0, 255] -&gt; [0.0,1.0]
    ]
)

and I confuse about how to put it with &quot;Data.DataLoader&quot;.
And I knew that there is a other way to change the training data from numpy to dataloader like this, and here is the link
train_transform = transforms.Compose([
    transforms.ToPILImage(),
    transforms.RandomHorizontalFlip(), 
    transforms.RandomRotation(15), 
    transforms.ToTensor(), 
])

test_transform = transforms.Compose([
    transforms.ToPILImage(),                                    
    transforms.ToTensor(),
])


class ImgDataset(Dataset):
    def __init__(self, x, y=None, transform=None):
        self.x = x
        self.y = y
        if y is not None:
            self.y = torch.LongTensor(y)
        self.transform = transform
    def __len__(self):
        return len(self.x)
    def __getitem__(self, index):
        X = self.x[index]
        if self.transform is not None:
            X = self.transform(X)
        if self.y is not None:
            Y = self.y[index]
            return X, Y
        else:
            return X

train_set = ImgDataset(train_x, train_y, train_transform)
val_set = ImgDataset(val_x, val_y, test_transform)
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False)

My way might be silly but I would like to try it, and any help would be appreciated. I hope I explain it well, and thanks advance.
Here is the complete code if needed:
import os
import numpy as np
import pandas as pd
import cv2
import torch
import torch.nn as nn
from torch.nn import functional as F
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch import optim
import pandas as pd
from torch.utils.data import DataLoader, Dataset
import torch.utils.data as Data



'''Initialize Params'''
epochs = 3
learning_rate = 0.0001
momentum = 0.5
batch_size = 128


transform = transforms.Compose([
    transforms.ToTensor(), # range [0, 255] -&gt; [0.0,1.0]
    ]
)



'''Load Data'''
def readFile(path,label):
    image_dir = sorted(os.listdir(path))
    # x stores photos
    x = np.zeros((len(image_dir),128,128,3),dtype=np.uint8)
    # y stores labels
    y = np.zeros((len(image_dir)), dtype=np.uint8)
    for i, file in enumerate(image_dir):
        img = cv2.imread(os.path.join(path, file))
        x[i, :, :] = cv2.resize(img,(128, 128))
        if label:
            y[i] = int(file.split(&quot;_&quot;)[0])
    if label:
        return x,y
    else:
        return x

train_x, train_y = readFile('./food/training',True)
val_x, val_y = readFile('./food/validation',True)
test_x = readFile('./food/testing',False)
# print(&quot;Reading data: &quot;)
# print(&quot;Size of training data = {}&quot;.format(len(train_x)))
# print(&quot;Size of validation data = {}&quot;.format(len(val_x)))
# print(&quot;Size of Testing data = {}&quot;.format(len(test_x)))


'''DataSet'''
train_x = torch.tensor(train_x)
# print(train_x.shape)
train_x = train_x.transpose(1,3).float()
train_y = torch.tensor(train_y)
val_x = torch.tensor(val_x)
val_x = val_x.transpose(1, 3).float()
val_y = torch.tensor(val_y)


train_dataset = Data.TensorDataset(train_x,train_y)
val_dataset = Data.TensorDataset(val_x,val_y)

train_loader = Data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
val_loader = Data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=True) 


'''Create Model'''
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # nn.Conv2d(input_channel, output_channel, kernel, stride)
        self.conv1 = nn.Conv2d(3,64,5,1,1) 
        nn.BatchNorm2d(64) 
        self.conv2 = nn.Conv2d(64,128,5,1,1)
        nn.BatchNorm2d(128)
        self.conv3 = nn.Conv2d(128,256,5,1,1)
        nn.BatchNorm2d(256)
        self.conv4 = nn.Conv2d(256,256,5,1,1)
        nn.BatchNorm2d(256)
        self.conv4_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(6*6*256, 1024)  
        # self.fc1 = nn.Linear(512*4*4, 1024)
        self.fc2 = nn.Linear(1024, 512)
        self.fc3 = nn.Linear(512, 256)
        self.fc4 = nn.Linear(256, 11)


    def forward(self, x):
        # maxpooling 1
        x = self.conv1(x)
        x = F.relu(x) # 124*124*64
        x = F.max_pool2d(x, 2) # 62*62*20

        # maxpooling 2
        x = self.conv2(x)
        x = F.relu(x) # 58*58*128
        x = F.max_pool2d(x, 2) # 29*29*40

        # maxpooling 3
        x = self.conv3(x)
        x = F.relu(x) # 25*25*256
        x = F.max_pool2d(x, 2) # 12*12*100

        # maxpooling 4
        x = self.conv4(x)
        x = F.relu(x) # 8*8*256
        x = F.max_pool2d(x, 2) # 4*4*256

        x = x.view(-1,6*6*256)
        # print(x.shape)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = self.fc4(x)
        # return F.log_softmax(x)
        return F.log_softmax(x)


'''Initialize the network'''
net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=learning_rate)


'''Train Model'''
if __name__ == '__main__':
    # print(train_x.shape)  torch.Size([9866, 128, 128, 3])
    device = torch.device(&quot;cuda:0&quot; if torch.cuda.is_available() else &quot;cpu&quot;)
    net.to(device)
    for epoch in range(epochs):
        for i,data in enumerate(train_loader):
            x,y = data
            # print(x)
            # print('--------------------')
            sum_loss = 0.0
            optimizer.zero_grad()  
            x,y = Variable(x).cuda(), Variable(y).cuda()
            outputs = net(x)  
            loss = criterion(outputs, y.long())  
            loss.backward()
            optimizer.step()  
            # print(loss.item())
            # sum_loss += loss.item()
            # if i % 500 == 99:
            #     print('[%d,%d] loss:%.03f' %
            #             (epoch + 1, i + 1, sum_loss / 500))
            #     sum_loss = 0.0


        '''Test Model with Validation'''
        net.eval() 
        correct = 0
        total = 0
        for val_data in val_loader:
            x,y = val_data
            x, y = Variable(x).cuda(), Variable(y).cuda()
            output_test = net(x)
            _, predicted = torch.max(output_test, 1)
            total += y.size(0)
            correct += (predicted == y).sum()
        print(&quot;correct1: &quot;, correct)
        print(&quot;Test acc: {0}&quot;.format(correct.item() /len(val_loader)))


The data normalization performed on the model's inputs (whether it'd be your training, validation, or testing set) depends on your training data statistics. More specifically, it corresponds to the channel-wise mean and standard deviation of the images used in the training set.

torchvision.transforms.Normalize
You can do so with torchvision's transforms.Normalize. It is a very easy function, yet a lot of people seem to have trouble understanding it - or fail to read its documentation...

torchvision.transforms.Normalize(mean, std, inplace=False) 
Normalize a tensor image with mean and standard deviation. This transform does not support PIL Image. Given mean: (mean[1], ..., mean[n]) and std: (std[1], ..., std[n]) for n channels, this transform will normalize each channel of the input torch.*Tensor i.e., output[channel] = (input[channel] - mean[channel]) / std[channel]

You can argue the naming for both arguments 'mean' and 'std' is inaccurate is rather misleading indeed. This can explain why it's so misused (today again).
It is nothing more than a channel-wise shift-scale operator.

Misusage of the transform
Very frequently I see this type of normalization setup.
T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))

To clarify, the above won't do the following two things:

it won't normalize your input to [0, 1].

it won't standardize your input (i.e. make mean=0, std=1)


The described transform will map values from [0, 1] to [-1, 1] (since (x - .5) / .5 = 2x - 1).
&gt;&gt;&gt; x = torch.tensor([0., .5, 1.]).reshape(1, 3, 1, 1)
tensor([0.0, 0.5, 1.0])

&gt;&gt;&gt; t = T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
&gt;&gt;&gt; t(x).flatten()
tensor([-1.,  0.,  1.])


How to properly use transforms.Normalize
In your case, you shouldn't use .5 as the mean and std parameters. This doesn't make any sense. If you're using a T.ToTensor prior to this you are essentially mapping your values to [-1, 1]...
Instead, you should initialize the transform with the actual dataset mean and std, the ones you measure from the dataset directly.
For instance, with Imagenet, the statistics are: [0.485, 0.456, 0.406] and [0.229, 0.224, 0.225] for mean and standard deviation respectively. You should be doing:
&gt;&gt;&gt; t = T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))

Having imported torchvision.transforms as T
