Neural networks pytorch
I am very new in pytorch and implementing my own network of image classifier. However I see for each epoch training accuracy is very good but validation accuracy is 0.i noted till 5th epoch. I am using Adam optimizer and have learning rate .001. also resampling the whole data set after each epoch into training n validation set. Please help where I am going wrong.

Here is my code:

### where is data?
data_dir_train = '/home/sup/PycharmProjects/deep_learning/CNN_Data/training_set'
data_dir_test = '/home/sup/PycharmProjects/deep_learning/CNN_Data/test_set'

# Define your batch_size
batch_size = 64

allData = datasets.ImageFolder(root=data_dir_train,transform=transformArr)


# We need to further split our training dataset into training and validation sets.
def split_train_validation():
    # Define the indices
    num_train = len(allData)
    indices = list(range(num_train)) # start with all the indices in training set
    split = int(np.floor(0.2 * num_train)) # define the split size
    #train_idx, valid_idx = indices[split:], indices[:split]

    # Random, non-contiguous split
    validation_idx = np.random.choice(indices, size=split, replace=False)
    train_idx = list(set(indices) - set(validation_idx))
    # define our samplers -- we use a SubsetRandomSampler because it will return
    # a random subset of the split defined by the given indices without replacement
    train_sampler = SubsetRandomSampler(train_idx)
    validation_sampler = SubsetRandomSampler(validation_idx)

    #train_loader = DataLoader(allData,batch_size=batch_size,sampler=train_sampler,shuffle=False,num_workers=4)
    #validation_loader = DataLoader(dataset=allData,batch_size=1, sampler=validation_sampler)

    return (train_sampler,validation_sampler)


Training

from torch.optim import Adam
import torch
import createNN
import torch.nn as nn
import loadData as ld
from torch.autograd import  Variable
from torch.utils.data import DataLoader

# check if cuda - GPU support available
cuda = torch.cuda.is_available()

#create model, optimizer and loss function
model = createNN.ConvNet(class_num=2)
optimizer = Adam(model.parameters(),lr=.001,weight_decay=.0001)
loss_func = nn.CrossEntropyLoss()

if cuda:
    model.cuda()

# function to save model
def save_model(epoch):
    torch.save(model.load_state_dict(),'imageClassifier_{}.model'.format(epoch))
    print('saved model at epoch',epoch)

def exp_lr_scheduler (  epoch , init_lr = args.lr, weight_decay = args.weight_decay, lr_decay_epoch = cf.lr_decay_epoch):
 lr = init_lr * ( 0.5 ** (epoch // lr_decay_epoch))

def train(num_epochs):
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('\n\nEpoch {}'.format(epoch))
        train_sampler, validation_sampler = ld.split_train_validation()
        train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
        validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
        model.train()
        acc = 0.0
        loss = 0.0
        total = 0
        # train model with training data
        for i,(images,labels) in enumerate(train_loader):
            # if cuda then move to GPU
            if cuda:
                images = images.cuda()
                labels = labels.cuda()
            # Variable class wraps a tensor and we can calculate grad
            images = Variable(images)
            labels = Variable(labels)
            # reset accumulated gradients for each batch
            optimizer.zero_grad()
            # pass images to model which returns preiction
            output = model(images)
            #calculate the loss based on prediction and actual
            loss = loss_func(output,labels)
            # backpropagate the loss and compute gradient
            loss.backward()
            # update weights as per the computed gradients
            optimizer.step()

            # prediction class
            predVal , predClass = torch.max(output.data, 1)
            acc += torch.sum(predClass == labels.data)
            loss += loss.cpu().data[0]
            total += labels.size(0)
        # print the statistics
        train_acc = acc/total
        train_loss = loss / total
        print('Mean train acc = {} over epoch = {}'.format(epoch,acc))
        print('Mean train loss = {} over epoch = {}'.format(epoch, loss))

        # Valid model with validataion data
        model.eval()
        acc = 0.0
        loss = 0.0
        total = 0
        for i,(images,labels) in enumerate(validation_loader):
            # if cuda then move to GPU
            if cuda:
                images = images.cuda()
                labels = labels.cuda()
            # Variable class wraps a tensor and we can calculate grad
            images = Variable(images)
            labels = Variable(labels)
            # reset accumulated gradients for each batch
            optimizer.zero_grad()
            # pass images to model which returns preiction
            output = model(images)
            #calculate the loss based on prediction and actual
            loss = loss_func(output,labels)
            # backpropagate the loss and compute gradient
            loss.backward()
            # update weights as per the computed gradients
            optimizer.step()

            # prediction class
            predVal, predClass = torch.max(output.data, 1)
            acc += torch.sum(predClass == labels.data)
            loss += loss.cpu().data[0]
            total += labels.size(0)
        # print the statistics
        valid_acc = acc / total
        valid_loss = loss / total
        print('Mean train acc = {} over epoch = {}'.format(epoch, valid_acc))
        print('Mean train loss = {} over epoch = {}'.format(epoch, valid_loss))

        if(best_acc&lt;valid_acc):
            best_acc = valid_acc
            save_model(epoch)

        # at 30th epoch we save the model
        if (epoch == 30):
            save_model(epoch)


train(20)


I think you did not take into account that acc += torch.sum(predClass == labels.data) returns a tensor instead of a float value. Depending on the version of pytorch you are using I think you should change it to:

acc += torch.sum(predClass == labels.data).cpu().data[0] #pytorch 0.3
acc += torch.sum(predClass == labels.data).item() #pytorch 0.4


Although your code seems to be working for old pytorch version, I would recommend you to upgrade to the 0.4 version.

Also, I mentioned other problems/typos in your code. 

You are loading the dataset for every epoch. 

for epoch in range(num_epochs):
    print('\n\nEpoch {}'.format(epoch))
    train_sampler, validation_sampler = ld.split_train_validation()
    train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
    validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
    ...


That should not happen, it should be enough loading it once

train_sampler, validation_sampler = ld.split_train_validation()
train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
for epoch in range(num_epochs):
    print('\n\nEpoch {}'.format(epoch))
    ...


In the training part you have (this does not happen in the validation):

train_acc = acc/total
train_loss = loss / total
print('Mean train acc = {} over epoch = {}'.format(epoch,acc))
print('Mean train loss = {} over epoch = {}'.format(epoch, loss))


Where you are printing acc instead of train_acc

Also, in the validation part I mentioned that you are printing print('Mean train acc = {} over epoch = {}'.format(epoch, valid_acc)) when it should be something like 'Mean val acc'.

Changing this lines of code, using a standard model I created and CIFAR dataset the training seems to converge, accuracy increases at every epoch while mean loss value decreases. 

I Hope I could help you!
