Why my CNN regressor doesn't work (Pytorch)
I'm trying to convert my tensorflow code to pytorch.
Simply speaking, it estimates 7 values (number) from images using CNN.(regressor)
The backbone network is vgg16 with pretrained weights, I'd like to convert last fcl (actually  due to ImageNet dataset, the last fcl output is 1000 classes), to (4096 x 4096), and add more fcls.
before :
vgg last fcl (4096 x 1000)
after:
vgg last fcl (change to 4096 x 4096)
----add fcl1 (4096 x 4096)
----add fcl2 (4096 x 2048)
└ add fclx (2048 x 3)
└ add fclq (2048 x 4)
: fcl2 is connected to two different tensors, with size of 3 and 4
Here, I tried to do it with only one image (for just debugging) and GT values (7 values) with L2 Loss.
If I do that using Tensorflow, the loss decreases drastically, and When I Infer an image, it gives almost similar values to GT.
However, If I try to do it using Pytorch, It looks like training doesn't work well.
I guess the loss should sharply decrease while training (almost for every iteration)
What's the problem?

The loss is actually |x-x'|^2 + b|q-q'|^2, well-known as L2-norm used in PoseNet(Kendall, 2015). x has three values of position and q has four values of quaternion(rotation). b is the hyperparameter determined by user.

from torchvision import models
import torch.nn as nn
import torch
from torch.autograd import Variable
import torch.optim as optim
import os
import os.path
import torch.utils.data as data
from torchvision import transforms as T
from PIL import Image

class DataSource(data.Dataset):
    def __init__(self, root, train=True, transforms=None, txtName='dataset_train'):
        self.root = os.path.expanduser(root)
        self.transforms = transforms
        self.train = train
        self.imageFormat = '.jpg'
        self.image_poses = []
        self.image_paths = []
        self.txtName = txtName
        self._get_data()

        if transforms is None:
            normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
                                    std=[0.229, 0.224, 0.225])
            if not train:
                self.transforms = T.Compose(
                    [T.Resize(256),
                     T.CenterCrop(224),
                     T.ToTensor(),
                     normalize]
                )
            else:
                self.transforms = T.Compose(
                    [T.Resize(256),
                     T.CenterCrop(224),
                     # T.RandomCrop(224),
                     T.ToTensor(),
                     normalize]
                )

    def _get_data(self):
        txt_file = self.root + '/' + self.txtName + '.txt'
        count = 0
        with open(txt_file, 'r') as f:
            for line in f:
                if len(line.split()) != 8:
                    next(f)
                fname, p0, p1, p2, p3, p4, p5, p6 = line.split()
                p0 = float(p0); p1 = float(p1); p2 = float(p2);
                p3 = float(p3); p4 = float(p4); p5 = float(p5); p6 = float(p6)
                ImageFullName = self.root + '/' +  fname
                if count == 0:
                    if os.path.isfile(ImageFullName) == False:
                        self.imageFormat = '.png'

                if self.imageFormat != '.jpg':
                    ImageFullName = ImageFullName.replace('.jpg', self.imageFormat)

                self.image_poses.append([p0, p1, p2, p3, p4, p5, p6])
                self.image_paths.append(ImageFullName)
                count += 1
        print('Total : ', len(self.image_paths), ' images')

    def __getitem__(self, index):
        img_path = self.image_paths[index]
        img_pose = self.image_poses[index]
        data = Image.open(img_path)
        data = self.transforms(data)
        return data, torch.tensor(img_pose)

    def __len__(self):
        return len(self.image_paths)

class PoseLoss(nn.Module):
    def __init__(self, beta, device = 'cuda'):
        super(PoseLoss, self).__init__()
        self.beta = beta
        self.device = device
        self.t_loss_fn = nn.MSELoss()

    def forward(self, x, q, poseGT):
        GT_x = poseGT[:, 0:3]
        GT_q = poseGT[:, 3:]

        xx = Variable(x, requires_grad=True).to(self.device)
        qq = Variable(q, requires_grad=True).to(self.device)
        print('GT', GT_x, GT_q)
        print('Estim', xx, qq)

        loss = torch.sqrt(self.t_loss_fn(GT_x[:, :3].cpu(), xx[:, :3].cpu())) + self.beta*torch.sqrt(self.t_loss_fn(GT_q[:, 3:].cpu(), qq[:, 3:].cpu()))
        return loss

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.backbone =  models.vgg16(pretrained=True)
        self.backbone._modules['classifier'][6] = nn.ReLU(nn.Linear(4096, 4096))
        self.fcl = nn.Sequential(nn.Linear(4096, 4096), nn.ReLU(), nn.Linear(4096, 2048), nn.ReLU())
        self.xyz = nn.Linear(2048, 3)
        self.q = nn.Linear(2048, 4)

    def forward(self, x):
        x1 = self.backbone(x)
        x2 = self.fcl(x1)
        xyz = self.xyz(x2)
        q = self.q(x2)
        return xyz, q

batch_size = 1
learning_rate = 10e-5
training_epochs = 100


if __name__ == &quot;__main__&quot;:
    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    data = DataSource(DatasetDirectory + DatasetFolder, train=True, transforms=None, txtName=TrainDatasetList)
    data_loader = torch.utils.data.DataLoader(dataset=data, batch_size=batch_size, shuffle=False, num_workers=4)

    model = Net().to(device)

    model.train()

    criterion = PoseLoss(beta = 100, device = device)

    optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas = (0.9, 0.999), eps =0.00000001)

    iteration = 0
    minloss = 10e8
    minlossindex = -1
    for epoch in range(1, training_epochs):
        dataiter = iter(data_loader)
        for Images, Poses in dataiter:
            optimizer.zero_grad()
            Images = Images.to(device).float()
            x, q = model(Images)
            loss = criterion(x, q, Poses)
            loss.backward()
            loss = loss.item()/ batch_size
            optimizer.step()
            print(epoch, ' : ', iteration , ' -&gt; ' , loss, ' minloss ', minloss, ' at ', minlossindex)
            if loss &lt; minloss:
                minloss = loss
                minlossindex = iteration
                if epoch &lt; (int)(training_epochs*0.8):
                    torch.save(model.state_dict(), 'Min.pth')
            iteration = iteration + 1
    torch.save(model.state_dict(), 'Fin.pth')


The estimated results tends to be zero for all 7 values, I cannot come up with why it gives such values.

Also, as I mentioned above, the loss values do not decrease dramatically while training(I expected It should be decreased dramatically for every iteration until it converges, because I used only one image for training)


If you move the data from gpu to cpu, you lose the historic in the computation graph and therefore the derivative is not propagated to the previous layers.
I do the following, usually the data is transferred to the device after sampling with the dataloader.
...
for Images, Poses in dataiter:
    Images = Images.to(device)
    Poses = Poses.to(device)
...

From here you will have all the data in gpu. Also, is not necessary to apply the variable in x and q. Automatically, when a layer is defined in pytorch it is already indicated that the tensor is a variable and that it must have an accumulation of gradient.
On the other hand, you don't need the sqrt in the loss either. Think that sqrt function is monomically increasing, so minimize mse is the same as minimize rmse. Putting the sqrt will probably make the training somewhat more unstable and will only be useful if you want to penalize in the same order of magnitude as the data.
