Instruction: RuntimeError due to inplace operation in GAN generator architecture with skip connections

Input:
I get the following error for a GAN model I am using to perform image colorization. It uses the LAB color space as is common in image colorization. The generator generates the a ad b channels for a given L channel. The discriminator is fed all three channels after concatenation.
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [64, 64, 128, 128]], which is output 0 of ReluBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).
I believe the error is due to the skip connections but I cannot quite put my finger on it. Any help would be appreciated!
Here is the model:
class NetGen(nn.Module):
    '''Generator'''
    def __init__(self):
        super(NetGen, self).__init__()

        self.conv1 = nn.Conv2d(1, 64, 3, stride=2, padding=1, bias=False)
        self.bnorm1 = nn.BatchNorm2d(64)
        self.relu1 = nn.LeakyReLU(0.1)

        self.conv2 = nn.Conv2d(64, 128, 3, stride=2, padding=1, bias=False)
        self.bnorm2 = nn.BatchNorm2d(128)
        self.relu2 = nn.LeakyReLU(0.1)

        self.conv3 = nn.Conv2d(128, 256, 3, stride=2, padding=1, bias=False)
        self.bnorm3 = nn.BatchNorm2d(256)
        self.relu3 = nn.LeakyReLU(0.1)

        self.conv4 = nn.Conv2d(256, 512, 3, stride=2, padding=1, bias=False)
        self.bnorm4 = nn.BatchNorm2d(512)
        self.relu4 = nn.LeakyReLU(0.1)

        self.conv5 = nn.Conv2d(512, 512, 3, stride=2, padding=1, bias=False)
        self.bnorm5 = nn.BatchNorm2d(512)
        self.relu5 = nn.LeakyReLU(0.1)

        self.deconv6 = nn.ConvTranspose2d(512, 512, 3, stride=2, padding=1, output_padding=1, bias=False)
        self.bnorm6 = nn.BatchNorm2d(512)
        self.relu6 = nn.ReLU()

        self.deconv7 = nn.ConvTranspose2d(512, 256, 3, stride=2, padding=1, output_padding=1, bias=False)
        self.bnorm7 = nn.BatchNorm2d(256)
        self.relu7 = nn.ReLU()

        self.deconv8 = nn.ConvTranspose2d(256, 128, 3, stride=2, padding=1, output_padding=1, bias=False)
        self.bnorm8 = nn.BatchNorm2d(128)
        self.relu8 = nn.ReLU()

        self.deconv9 = nn.ConvTranspose2d(128, 64, 3, stride=2, padding=1, output_padding=1, bias=False)
        self.bnorm9 = nn.BatchNorm2d(64)
        self.relu9 = nn.ReLU()

        self.deconv10 = nn.ConvTranspose2d(64, 2, 3, stride=2, padding=1, output_padding=1, bias=False)
        self.tanh = nn.Tanh()

    def forward(self, x):
        h = x
        h = self.conv1(h)
        h = self.bnorm1(h)
        h = self.relu1(h)
        pool1 = h

        h = self.conv2(h)
        h = self.bnorm2(h)
        h = self.relu2(h)
        pool2 = h

        h = self.conv3(h)
        h = self.bnorm3(h)
        h = self.relu3(h)
        pool3 = h

        h = self.conv4(h)
        h = self.bnorm4(h)
        h = self.relu4(h)
        pool4 = h

        h = self.conv5(h)
        h = self.bnorm5(h)
        h = self.relu5(h)

        h = self.deconv6(h)
        h = self.bnorm6(h)
        h = self.relu6(h)
        h += pool4

        h = self.deconv7(h)
        h = self.bnorm7(h)
        h = self.relu7(h)
        h += pool3

        h = self.deconv8(h)
        h = self.bnorm8(h)
        h = self.relu8(h)
        h += pool2

        h = self.deconv9(h)
        h = self.bnorm9(h)
        h = self.relu9(h)
        h += pool1

        h = self.deconv10(h)
        h = self.tanh(h)
        return h

class NetDis(nn.Module):
    '''Discriminator'''
    def __init__(self):
        super(NetDis, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(3, 64, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(64),
            nn.LeakyReLU(0.1),

            nn.Conv2d(64, 128, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.1),

            nn.Conv2d(128, 256, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.1),

            nn.Conv2d(256, 512, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.1),

            nn.Conv2d(512, 512, 3, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.1),

            nn.Conv2d(512, 512, 8, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.1),

            nn.Conv2d(512, 1, 1, stride=1, padding=0, bias=False),
            nn.Sigmoid()
        )

    def forward(self, x):
        return self.main(x)

Here is the weight init function:
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1.0, 0.02)
        nn.init.constant_(m.bias.data, 0)

Here is the training and validation code:
class Trainer:
    def __init__(self, epochs, batch_size, learning_rate, num_workers):
        self.epochs = epochs
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.num_workers = num_workers
        self.train_paths = train_paths
        self.val_paths = val_paths
        self.real_label = 1
        self.fake_label = 0

    def train(self):
        train_dataset = ColorizeData(paths=self.train_paths)
        train_dataloader = DataLoader(train_dataset, batch_size=self.batch_size, num_workers=self.num_workers,pin_memory=True, drop_last = True)
        # Model
        model_G = NetGen().to(device)
        model_D = NetDis().to(device)

        model_G.apply(weights_init)
        model_D.apply(weights_init)

        optimizer_G = torch.optim.Adam(model_G.parameters(),
                             lr=self.learning_rate, betas=(0.5, 0.999),
                             eps=1e-8, weight_decay=0)
        optimizer_D = torch.optim.Adam(model_D.parameters(),
                             lr=self.learning_rate, betas=(0.5, 0.999),
                             eps=1e-8, weight_decay=0)

        criterion = nn.BCELoss()
        L1 = nn.L1Loss()

        model_G.train()
        model_D.train()

        # train loop
        for epoch in range(self.epochs):
            print("Starting Training Epoch " + str(epoch + 1))
            for i, data in enumerate(tqdm(train_dataloader)):
                inputs, input_ab, input_l = data
                inputs = inputs.to(device)
                input_ab = input_ab.to(device)
                input_l = input_l.to(device)

                model_D.zero_grad()
                label = torch.full((self.batch_size,), self.real_label, dtype=torch.float, device=device)
                output = model_D(torch.cat([input_l, input_ab], dim=1))
                errD_real = criterion(torch.squeeze(output), label)
                errD_real.backward()

                fake = model_G(input_l)
                label.fill_(self.fake_label)

                output = model_D(torch.cat([input_l, fake.detach()], dim=1))
                errD_fake = criterion(torch.squeeze(output), label)
                errD_fake.backward()
                errD = errD_real + errD_fake
                optimizer_D.step()

                model_G.zero_grad()
                label.fill_(self.real_label)
                output = model_D(torch.cat([input_l, fake], dim=1))
                errG = criterion(torch.squeeze(output), label)
                errG_L1 = L1(fake.view(fake.size(0),-1), input_ab.view(input_ab.size(0),-1))
                errG = errG + 100 * errG_L1
                errG.backward()
                optimizer_G.step()

            print(f'Training: Epoch {epoch + 1} \t\t Discriminator Loss: {\
                errD / len(train_dataloader)}  \t\t Generator Loss: {\
                errG / len(train_dataloader)}')

            if (epoch + 1) % 1 == 0:
                errD_val, errG_val, val_len = self.validate(model_D, model_G, criterion, L1)
                print(f'Validation: Epoch {epoch + 1} \t\t Discriminator Loss: {\
                        errD_val / val_len}  \t\t Generator Loss: {\
                        errG_val / val_len}')

            torch.save(model_G.state_dict(), '../Results/Model_GAN/Generator/saved_model_' + str(epoch + 1) + '.pth')
            torch.save(model_D.state_dict(), '../Results/Model_GAN/Discriminator/saved_model_' + str(epoch + 1) + '.pth')

    def validate(self, model_D, model_G, criterion, L1):

        model_G.eval()
        model_D.eval()
        with torch.no_grad():
            valid_loss = 0.0
            val_dataset = ColorizeData(paths=self.val_paths)
            val_dataloader = DataLoader(val_dataset, batch_size=self.batch_size, num_workers=self.num_workers, pin_memory=True, drop_last = True)
            for i, data in enumerate(val_dataloader):
                inputs, input_ab, input_l = data
                inputs = inputs.to(device)
                input_ab = input_ab.to(device)
                input_l = input_l.to(device)

                label = torch.full((self.batch_size,), self.real_label, dtype=torch.float, device=device)
                output = model_D(torch.cat([input_l, input_ab], dim=1))
                errD_real = criterion(torch.squeeze(output), label)

                fake = model_G(input_l)
                label.fill_(self.fake_label)
                output = model_D(torch.cat([input_l, fake.detach()], dim=1))
                errD_fake = criterion(torch.squeeze(output), label)

                errD = errD_real + errD_fake

                label.fill_(self.real_label)
                output = model_D(torch.cat([input_l, fake], dim=1))
                errG = criterion(torch.squeeze(output), label)
                errG_L1 = L1(fake.view(fake.size(0),-1), input_ab.view(input_ab.size(0),-1))
                errG = errG + 100 * errG_L1

        return errD, errG, len(val_dataloader)

EDIT
As suggested by @manaclan here is the code I use to run the pipeline:
trainer = Trainer(epochs = 100, batch_size = 64, learning_rate = 0.0002, num_workers = 2)
trainer.train()

Here is the data loader:
class ColorizeData(Dataset):
    def __init__(self, paths):
        self.input_transform = T.Compose([T.ToTensor(),
                                          T.Resize(size=(256,256)),
                                          T.Grayscale(),
                                          T.Normalize((0.5), (0.5))
                                          ])
        self.lab_transform = T.Compose([T.ToTensor(),
                                          T.Resize(size=(256,256)),
                                          T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
                                          ])
        self.paths = paths

    def __len__(self) -> int:
        return len(self.paths)

    def __getitem__(self, index: int) -> Tuple[torch.Tensor, torch.Tensor]:
        image = Image.open(self.paths[index]).convert("RGB")
        input_image = self.input_transform(image)
        image_lab = rgb2lab(image)
        image_lab = self.lab_transform(image_lab)
        image_l = image_lab[0, :, :]
        image_ab = image_lab[1:3, :, :]
        return (input_image.float(), image_ab.float(), image_l.float().reshape(1, 256, 256))

Here are the imports:
from typing import Tuple
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as T
import torch
import numpy as np
import os
import torch.nn as nn
import torchvision.models as models
import torchvision
import torch.nn.functional as functional
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
from PIL import Image
import glob
import matplotlib.pyplot as plt
from tqdm import tqdm
from skimage.color import lab2rgb, rgb2lab, rgb2gray
from skimage import io

from torchvision.transforms.functional import resize

To reproduce the error, just use any dataset of color images.
I have the following code to get my train, test, and validation images from the folder "Dataset":
path = "../Dataset/"
paths = np.array(glob.glob(path + "/*.jpg"))
rand_indices = np.random.permutation(len(paths))          # Number of images in dataset
train_indices, val_indices, test_indices = rand_indices[:3600], rand_indices[3600:4000], rand_indices[4000:]
train_paths = paths[train_indices]
val_paths = paths[val_indices]
test_paths = paths[test_indices]

NOTE: I am using Google Colab, maybe this might be a potential problem? Also, I am using torch version 1.10.0+cu111.
I did use a sequential model without skip connections for the generator before this, and I did not have this error then.

Output: So apparently, the problem is the inplace skip connection written as h += poolX. Writing this update out of place as h = h + poolX fixed it. h is needed for gradient calculation in some layers, so inplace modification will mess it up.


Instruction: How to clear GPU memory after using model?

Input:
I'm trying to free up GPU memory after finishing using the model.

I checked the nvidia-smi before creating and trainning the model: 402MiB /  7973MiB
After creating and training the model, I checked again the GPU memory status with  nvidia-smi: 7801MiB /  7973MiB
Now I tried to free up GPU memory with:

del model
torch.cuda.empty_cache()
gc.collect()

and checked again the GPU memory:  2361MiB /  7973MiB

As you can see not all the GPU memory was released (I expected to get 400~MiB / 7973MiB).
I can only relase the GPU memory via terminal (sudo fuser -v /dev/nvidia*  and kill pid)

Is there a way to free up the GPU memory after I done using the model ?

Output:
This happens becauce pytorch reserves the gpu memory for fast memory allocation. To learn more about it, see pytorch memory management. To solve this issue, you can use the following code:
from numba import cuda
cuda.select_device(your_gpu_id)
cuda.close()

However, this comes with a catch. It closes the GPU completely. So, you can't start training without restarting everything.


Instruction: How unfold operation works in pytorch with dilation and stride?

Input:
In my case I am applying this unfold operation on a tensor of A as given below:
A.shape=torch.Size([16, 309,128])
A = A.unsqueeze(1) # that's I guess for making it 4 dim for unfold operation
A_out= F.unfold(A, (7, 128), stride=(1,128),dilation=(3,1))
A_out.shape=torch.Size([16, 896,291])

I am not getting this 291. If the dilation factor is not there, it would be [16,896,303] right?
But if dialtion=3 then it's 291 how? Also here stride is not mentioned so deafualt is 1 but what if it is also mentioned like 4. Please guide.

Output:
Also here stride is not mentioned so default is 1 but what if it is
also mentioned like 4.

Your code already has stride=(1,128). If stride is only set to 4 it will be used like (4,4) in this case. This can be easily verified with formula below.

If the dilation factor is not there, it would be [16,896,303] right?

Yes. Example below.

But if dialtion=3 then it's 291 how?

Following the formula given in pytorch docs it comes to 291. After doing A.unsqueeze(1) the shape becomes, [16, 1, 309, 128]. Here, N=16, C=1,  H=309, W=128.

The output dimension is, (N, C * product(kernel_size), L). With kernel_size=(7,128) So this becomes, (16, 1 * 7 * 128, L) = (16, 896, L).
L can be calculated using the formula below with multiplication over each dimension.
L = d3 * d4

Over height dimension spatial_size[3] = 309, padding[3] = 0 default, dilation[3] = 3, kernel_size[3] = 7, stride[3] = 1.
d3 = (309 + 2 * 0 - 3 * (7 - 1) - 1) / 1 + 1
   = 291

Over width dimension spatial_size[4] = 128, padding[4] = 0 default, dilation[4] = 1, kernel_size[4] = 128, stride[4] = 128.
d4 = (128 + 2 * 0 - 1 * (128 - 1) - 1) / 128 + 1
   = 1

So, using above formula L becomes 291.
Code
import torch
from torch.nn import functional as F

A = torch.randn([16, 309,128])
print(A.shape)
A = A.unsqueeze(1)
print(A.shape)
A_out= F.unfold(A, kernel_size=(7, 128), stride=(1,128),dilation=(3,1))
print(A_out.shape)

Output
torch.Size([16, 309, 128])
torch.Size([16, 1, 309, 128])
torch.Size([16, 896, 291])

Links

https://pytorch.org/docs/stable/generated/torch.nn.Unfold.html
https://pytorch.org/docs/stable/generated/torch.nn.functional.unfold.html


Instruction: How can I make a filter in pytorch conv2d

Input:
I am really new to pytorch, and I've been making code convolution myself.
To apply convolution on input data, I use conv2d.
In the documentation,
torch.nn.Conv2d(in_channels, out_channels, kernel_size ...)

But where is a filter? To convolute, we should do it on input data with kernel. But there is only kernel size, not the elements of the kernel.
For example, There is an input data 5x5 and with 2x2 kernel with all 4 kernel's elements are 1 then I can make 4x4 output. So where can I put the elements of kernel?

Output:
The filter weights can be accessed using the weight parameter of the Conv2d object. For example,
>>> c = torch.nn.Conv2d(in_channels=2, out_channels=2, kernel_size=3)
>>> c.weight
Parameter containing:
tensor([[[[ 0.2156,  0.0930, -0.2319],
      [ 0.1333, -0.0846,  0.1848],
      [ 0.0765, -0.1799, -0.1273]],

     [[ 0.1173,  0.1650, -0.0876],
      [-0.1353,  0.0616, -0.1136],
      [-0.2326, -0.1509,  0.0651]]],

    [[[-0.2026,  0.2210,  0.0409],
      [-0.0818,  0.0793,  0.1074],
      [-0.1430, -0.0118, -0.2100]],

     [[-0.2025, -0.0508, -0.1731],
      [ 0.0217, -0.1616,  0.0702],
      [ 0.1903, -0.1864,  0.1523]]]], requires_grad=True)

The weights are initialized by default by sampling from a uniform distribution. You can also initialize weights using various weight initialization schemes.
If you want to manually change the weights, you can do it by modifying the weight parameter directly. For example, to set all the weights to 1, use,
>>> c.weight.data = torch.ones_like(c.weight)
>>> c.weight
Parameter containing:
tensor([[[[1., 1., 1.],
      [1., 1., 1.],
      [1., 1., 1.]],

     [[1., 1., 1.],
      [1., 1., 1.],
      [1., 1., 1.]]],

    [[[1., 1., 1.],
      [1., 1., 1.],
      [1., 1., 1.]],

     [[1., 1., 1.],
      [1., 1., 1.],
      [1., 1., 1.]]]], requires_grad=True)

Note that during training, the convolutional layers are typically a part of the computational graph, and their weights get automatically updated when making a backward() call.


Instruction: PyTorch Tensor broadcasting

Input:
I'm trying to figure out how to do the following broadcast:
I have two tensors, of sizes (n1,N) and (n2,N)
What I want to do is to multiply each row of the first tensor, with each row of the second tensor, and then sum each of there multiplied row result, so that my final tensor should be of the form (n1,n2).
I tried this:
x1*torch.reshape(x2,(x2.size(dim=0),x2.size(dim=1),1))

But obviously this doesn't work.. Can't figure out how to do this

Output:
What you describe seems to be effectively the same as performing a matrix multiplication between the first tensor and the transpose of the second tensor. This can be done as:
torch.matmul(x1, x2.T)


Instruction: A question about validation process in PyTorch: val_loss lower than train_loss

Input:
Is it possible that running my deep-learning, at some point during training it happens that my validation loss becomes lower than my training loss? I'm attaching my code for the training process:
def train_model(model, train_loader,val_loader,lr):

    "Model training"

    epochs=100

    model.train()

    train_losses = []

    val_losses = []

    criterion = nn.MSELoss()

    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=1e-5)

    #Reduce learning rate if no improvement is observed after 10 Epochs.

    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2, verbose=True)

    for epoch in range(epochs):

        for data in train_loader:

            y_pred = model.forward(data)

            loss1 = criterion(y_pred[:, 0], data.y[0])

            loss2 = criterion(y_pred[:,1], data.y[1])

            train_loss = 0.8*loss1+0.2*loss2

            optimizer.zero_grad()

            train_loss.backward()

            optimizer.step()

        train_losses.append(train_loss.detach().numpy())

        with torch.no_grad():

            for data in val_loader:

                y_val = model.forward(data)

                loss1 = criterion(y_val[:,0], data.y[0])

                loss2 = criterion(y_val[:,1], data.y[1])

                val_loss = 0.8*loss1+0.2*loss2

            #scheduler.step(loss)

        val_losses.append(val_loss.detach().numpy())

        print(f'Epoch: {epoch}, train_loss: {train_losses[epoch]:.3f} , val_loss: {val_losses[epoch]:.3f}')

    return train_losses, val_losses

It is a multi-task model where I compute separately the two losses and then consider a weighted sum.
What I'm not sure about is the indentation of val_loss that may cause some issues when printing out. In general I would say that I have some perplexities about validation:
1)First I pass all the batches that I have in my train_loader and adjust the training loss.
2)Then, I start iterating over my val_loader to make predictions on single batches of unseen data, but what I append in the val_losses list is the validation loss computed by the model on the last batch inside val_loader. I'm not sure if this is correct.
I'm attaching the printed train and val losses during training:
Epoch: 0, train_loss: 7.315 , val_loss: 7.027
Epoch: 1, train_loss: 7.227 , val_loss: 6.943
Epoch: 2, train_loss: 7.129 , val_loss: 6.847
Epoch: 3, train_loss: 7.021 , val_loss: 6.741
Epoch: 4, train_loss: 6.901 , val_loss: 6.624
Epoch: 5, train_loss: 6.769 , val_loss: 6.493
Epoch: 6, train_loss: 6.620 , val_loss: 6.347
Epoch: 7, train_loss: 6.452 , val_loss: 6.182
Epoch: 8, train_loss: 6.263 , val_loss: 5.996
Epoch: 9, train_loss: 6.051 , val_loss: 5.788
Epoch: 10, train_loss: 5.814 , val_loss: 5.555
Epoch: 11, train_loss: 5.552 , val_loss: 5.298
Epoch: 12, train_loss: 5.270 , val_loss: 5.022
Epoch: 13, train_loss: 4.972 , val_loss: 4.731
Epoch: 14, train_loss: 4.666 , val_loss: 4.431
Epoch: 15, train_loss: 4.357 , val_loss: 4.129
Epoch: 16, train_loss: 4.049 , val_loss: 3.828
Epoch: 17, train_loss: 3.752 , val_loss: 3.539
Epoch: 18, train_loss: 3.474 , val_loss: 3.269
Epoch: 19, train_loss: 3.220 , val_loss: 3.023
Epoch: 20, train_loss: 2.992 , val_loss: 2.803
Epoch: 21, train_loss: 2.793 , val_loss: 2.613
Epoch: 22, train_loss: 2.626 , val_loss: 2.453
Epoch: 23, train_loss: 2.488 , val_loss: 2.323
Epoch: 24, train_loss: 2.378 , val_loss: 2.220
Epoch: 25, train_loss: 2.290 , val_loss: 2.140
Epoch: 26, train_loss: 2.221 , val_loss: 2.078
Epoch: 27, train_loss: 2.166 , val_loss: 2.029
Epoch: 28, train_loss: 2.121 , val_loss: 1.991
Epoch: 29, train_loss: 2.084 , val_loss: 1.959
Epoch: 30, train_loss: 2.051 , val_loss: 1.932
Epoch: 31, train_loss: 2.022 , val_loss: 1.909
Epoch: 32, train_loss: 1.995 , val_loss: 1.887
Epoch: 33, train_loss: 1.970 , val_loss: 1.867
Epoch: 34, train_loss: 1.947 , val_loss: 1.849
Epoch: 35, train_loss: 1.924 , val_loss: 1.831
Epoch: 36, train_loss: 1.902 , val_loss: 1.815
Epoch: 37, train_loss: 1.880 , val_loss: 1.799
Epoch: 38, train_loss: 1.859 , val_loss: 1.783
Epoch: 39, train_loss: 1.839 , val_loss: 1.769
Epoch: 40, train_loss: 1.820 , val_loss: 1.755
Epoch: 41, train_loss: 1.800 , val_loss: 1.742
Epoch: 42, train_loss: 1.781 , val_loss: 1.730
Epoch: 43, train_loss: 1.763 , val_loss: 1.717
Epoch: 44, train_loss: 1.744 , val_loss: 1.705
Epoch: 45, train_loss: 1.726 , val_loss: 1.694
Epoch: 46, train_loss: 1.708 , val_loss: 1.683

...

So I have the suspect that I'm messing up with indentation..

Output:
Validation loss can be lower than the training loss.
As you mentioned in point 2, you are only storing/appending the train and validation loss on the last batch. This may not be what you want, and you may want to store the training loss at each iteration and look at its average value at the end. That would give you a better idea of training progress as that would be the loss on the whole data and single batch


Instruction: How to find argmax/argmin in only selected indices of a Pytorch tensor

Input:
I have a distance tensor
tensor([ 5,  10,  2,  3,  4], device='cuda:0')

And a indices tensor
tensor([ 0,  2,  3], device='cuda:0')

I want to find argmax of the distance tensor but only on the subset of indices specified by the indices tensor.
In this example, I would be looking at 0th, 2nd and 3rd elements of distance tensor (values 5, 2, 3) and returning the index 0 (the biggest value - 5 is on the 0th place in the distance tensor)
tensor([ 0], device='cuda:0')

Is something like this feasible without the use of for cycles?
Thanks

Output:
Here an example. You can check that the maximum dist value for the selected subset of items is at index zero, and the final output tensor contains value zero too. Note that as we are using 1D tensors, dim argument in torch.index_select is zero.
import torch

dist = torch.randn(5, 1)
#tensor([[ 0.3392],
#        [ 0.4472],
#        [ 0.1398],
#        [-1.0379],
#        [ 0.2950]])

idx = torch.tensor([0,2,3])
#tensor([0, 2, 3])

Just using max function and tensor filtering:
max_val = torch.max(torch.index_select(dist, 0, idx)).item()
#0.33918169140815735
(dist == max_val).nonzero(as_tuple=True)[0]
#tensor([0])


Instruction: how to convert a python list of lists to tensor using pytorch

Input:
I got a list that contains lists of different length. How can i transform it in a tensor in pytorch without using padding? Is it possible?
[[3, 5, 10, 11], [1, 5, 10]]

Output:
It depends on what you want to achieve with the data structure.
You can use torch.sparse, for example:
ll = [[3, 5, 10, 11], [1, 5, 10]]
n = len(ll)
m = max(len(l) for l in ll)

ids = [[], []]
values = []
for i, l in enumerate(ll):
    length = len(l)
    ids[0] += [i] * length  # rows
    ids[1] += list(range(length))  # cols
    values += l

t = torch.sparse_coo_tensor(ids, values, (n, m))

Otherwise, you can try with embedding techniques for corpus of documents, such as bag-of-words (though it will generate still some "padding"), tf-idf, etc.
bag-of-words with possible duplicates in inner lists
corpus = [[3, 5, 10, 11], [1, 5, 10]]
n = len(corpus)
m = max(max(inner) for inner in corpus)
t = torch.zeros(n, m)

for i, doc in enumerate(corpus):
    torch.bincount(corpus)

bag-of-words with distinct values in inner lists
corpus = [[3, 5, 10, 11], [1, 5, 10]]
n = len(corpus)
m = max(max(inner) for inner in corpus)

t = torch.zeros(n, m)
for i, doc in enumerate(corpus):
    t[i, doc] = 1


Instruction: Is the rotation matrix in Pybullet converting world coordinates into camera or camera to world?

Input:
I am working on a project, where I need to replace the renderings by pybullet with renders generated with pytorch3d.
I figured out that pybullet and pytorch3d have different definitions for the coordinate systems (see these links: pybullet, pytorch3d; x and z axes are flipped), and I accounted for that in my code. But I still have inconsistency in the rendered objects. I thought the problem could be that while pytorch3d expects a c2w rotation matrix (i.e. camera to world), pybullet could probably expect a w2c rotation matrix. However, I cannot find any documentation related to this. Has anyone ever encountered this problem, or maybe can give some useful hint on how to find out what exactly pybullet expects its rotation matrix to be?
Thanks!

Output:
I assume you are talking about the viewMatrix expected by pybullet.getCameraImage().  This should indeed be a world-to-camera rotation matrix.
However, in pyBullet the camera is looking in negative z-direction while I usually expect it to be in positive one.  I am compensating for this by adding a 180°-rotation around the x-axis:
rot_x_180 = np.array(
    [
        [1, 0, 0, 0],
        [0, -1, 0, 0],
        [0, 0, -1, 0],
        [0, 0, 0, 1],
    ]
)
tf_mat = rot_x_180 @ tf_world_to_camera
view_matrix = tf_mat.flatten(order="F")

where tf_world_to_camera is a homogeneous rotation matrix.


Instruction: CUDA OOM - But the numbers don't add upp?

Input:
I am trying to train a model using PyTorch. When beginning model training I get the following error message:
RuntimeError: CUDA out of memory. Tried to allocate 5.37 GiB (GPU 0; 7.79 GiB total capacity; 742.54 MiB already allocated; 5.13 GiB free; 792.00 MiB reserved in total by PyTorch)
I am wondering why this error is occurring. From the way I see it, I have 7.79 GiB total capacity. The numbers it is stating (742 MiB + 5.13 GiB + 792 MiB) do not add up to be greater than 7.79 GiB. When I check nvidia-smi I see these processes running
|    0   N/A  N/A      1047      G   /usr/lib/xorg/Xorg                168MiB |
|    0   N/A  N/A      5521      G   /usr/lib/xorg/Xorg                363MiB |
|    0   N/A  N/A      5637      G   /usr/bin/gnome-shell              161MiB |

I realize that summing all of these numbers might cut it close (168 + 363 + 161 + 742 + 792 + 5130 = 7356 MiB) but this is still less than the stated capacity of my GPU.

Output:
This is more of a comment, but worth pointing out.
The reason in general is indeed what talonmies commented, but you are summing up the numbers incorrectly. Let's see what happens when tensors are moved to GPU (I tried this on my PC with RTX2060 with 5.8G usable GPU memory in total):
Let's run the following python commands interactively:
Python 3.8.10 (default, Sep 28 2021, 16:10:42)
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> a = torch.zeros(1).cuda()
>>> b = torch.zeros(500000000).cuda()
>>> c = torch.zeros(500000000).cuda()
>>> d = torch.zeros(500000000).cuda()

The following are the outputs of watch -n.1 nvidia-smi:
Right after torch import:
|    0   N/A  N/A      1121      G   /usr/lib/xorg/Xorg                  4MiB |

Right after the creation of a:
|    0   N/A  N/A      1121      G   /usr/lib/xorg/Xorg                  4MiB |
|    0   N/A  N/A     14701      C   python                           1251MiB |

As you can see, you need 1251MB to get pytorch to start using CUDA, even if you only need a single float.
Right after the creation of b:
|    0   N/A  N/A      1121      G   /usr/lib/xorg/Xorg                  4MiB |
|    0   N/A  N/A     14701      C   python                           3159MiB |

b needs 500000000*4 bytes = 1907MB, this is the same as the increment in memory used by the python process.
Right after the creation of c:
|    0   N/A  N/A      1121      G   /usr/lib/xorg/Xorg                  4MiB |
|    0   N/A  N/A     14701      C   python                           5067MiB |

No surprise here.
Right after the creation of d:
|    0   N/A  N/A      1121      G   /usr/lib/xorg/Xorg                  4MiB |
|    0   N/A  N/A     14701      C   python                           5067MiB |

No further memory allocation, and the OOM error is thrown:
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
RuntimeError: CUDA out of memory. Tried to allocate 1.86 GiB (GPU 0; 5.80 GiB total capacity; 3.73 GiB already allocated; 858.81 MiB free; 3.73 GiB reserved in total by PyTorch)

Obviously:

The "already allocated" part is included in the "reserved in total by PyTorch" part. You can't sum them up, otherwise the sum exceeds the total available memory.
The minimum memory required to get pytorch running on GPU (1251M) is not included in the "reserved in total" part.

So in your case, the sum should consist of:

792MB (reserved in total)
1251MB (minimum to get pytorch running on GPU, assuming this is the same for both of us)
5.13GB (free)
168+363+161=692MB (other processes)

They sum up to approximately 7988MB=7.80GB, which is exactly you total GPU memory.


Instruction: Piecewise Activation Function

Input:
I'm trying to write a piecewise activation function whose slope between -6 and 0 is 0.1 and the other places are one.
And the input(X) size is (B, C, H, W).
So I concluded that the best way is the simple line code:
 x[-6<x and x<0] = x[-6<x and x<0] * 0.1

But I face this error:
RuntimeError: bool value of Tensor with more than one value is ambiguous

Is there any solution for solving this error?

Output:
The most simple version of what you need is:
import torch

def custom_activ(input):
    return torch.where((input>-6) & (input<0.) , 0.1*input, input)


Instruction: Python: Vectorize Calculation Implemented using Iterative Approach

Input:
I'm trying to implement some calculation, but I can't figure how to vectorize my code and not using loops.
Let me explain: I have a matrix M[N,C] of either 0 or 1. Another matrix Y[N,1] containing values of [0,C-1] (My classes). Another matrix ds[N,M] which is my dataset.
My output matrix is of size grad[M,C] and should be calculated as follow: I'll explain for grad[:,0], same logic for any other column.
For each row(sample) in ds, if Y[that sample] != 0 (The current column of output matrix) and M[that sample, 0] > 0 , then grad[:,0] += ds[that sample]
If Y[that sample] == 0, then grad[:,0] -= (ds[that sample] * <Num of non zeros in M[that sample,:]>)
Here is my iterative approach:
    for i in range(M.size(dim=1)):
        for j in range(ds.size(dim=0)):
            if y[j] == i:
                grad[:,i] = grad[:,i] - (ds[j,:].T * sum(M[j,:]))
            else:
                if M[j,i] > 0:
                    grad[:,i] = grad[:,i] + ds[j,:].T

Output:
Since you are dealing with three dimensions n, m, and c (in lowercase to avoid ambiguity), it can be useful to change the shape of all your tensors to (n, m, c), by replicating their values over the missing dimension (e.g. M(m, c) becomes M(n, m, c)).
However, you can skip the explicit replication and use broadcasting, so it is sufficient to unsqueeze the missing dimension (e.g. M(m, c) becomes M(1, m, c).
Given these considerations, the vectorization of your code becomes as follows
cond = y.unsqueeze(2) == torch.arange(M.size(dim=1)).unsqueeze(0)
pos = ds.unsqueeze(2) * M.unsqueeze(1) * cond
neg = ds.unsqueeze(2) * M.unsqueeze(1).sum(dim=0, keepdim=True) * ~cond
grad += (pos - neg).sum(dim=0)

Here is a small test to check the validity of the solution
import torch

n, m, c = 11, 5, 7

y = torch.randint(c, size=(n, 1))
ds = torch.rand(n, m)
M = torch.randint(2, size=(n, c))
grad = torch.rand(m, c)

def slow_grad(y, ds, M, grad):
    for i in range(M.size(dim=1)):
        for j in range(ds.size(dim=0)):
            if y[j] == i:
                grad[:,i] = grad[:,i] - (ds[j,:].T * sum(M[j,:]))
            else:
                if M[j,i] > 0:
                    grad[:,i] = grad[:,i] + ds[j,:].T
    return grad

def fast_grad(y, ds, M, grad):
    cond = y.unsqueeze(2) == torch.arange(M.size(dim=1)).unsqueeze(0)
    pos = ds.unsqueeze(2) * M.unsqueeze(1) * cond
    neg = ds.unsqueeze(2) * M.unsqueeze(1).sum(dim=0, keepdim=True) * ~cond
    grad += (pos - neg).sum(dim=0)
    return grad

# Assert equality of all elements function outputs, throws an exception if false
assert torch.all(slow_grad(y, ds, M, grad) == fast_grad(y, ds, M, grad))

Feel free to test on other cases as well!


Instruction: Is there a way to optimize the calculation of Bernoulli Log-Likelihoods for many multivariate samples?

Input:
I currently have two Torch Tensors, p and x, which both have the shape of (batch_size, input_size).
I would like to calculate the Bernoulli log likelihoods for the given data, and return a tensor of size (batch_size)
Here's an example of what I'd like to do:
I have the formula for log likelihoods of Bernoulli Random variables:
\sum_i^d x_{i} ln(p_i) + (1-x_i) ln (1-p_i)
Say I have p Tensor:
[[0.6 0.4 0], [0.33 0.34 0.33]]
And say I have the x tensor for the binary inputs based on those probabilities:
[[1 1 0], [0 1 1]]
And I want to calculate the log likelihood for every sample, which would result in:
[[ln(0.6)+ln(0.4)], [ln(0.67)+ln(0.34)+ln(0.33)]]
Would it be possible to do this computation without the use of for loops?
I know I could use torch.sum(axis=1) to do the final summation between the logs, but is it possible to do the Bernoulli log-likelihood computation without the use of for loops? or use at most 1 for loop? I am trying to vectorize this operation as much as possible. I could've sworn we could use LaTeX for equations before, did something change or is it another website?

Output:
Though not a good practice, you can directly use the formula on the tensors as follows (works because these are element wise operations):
import torch
p = torch.tensor([
    [0.6, 0.4, 0],
    [0.33, 0.34, 0.33]
])

x = torch.tensor([
    [1., 1, 0],
    [0, 1, 1]
])

eps = 1e-8
bll1 = (x * torch.log(p+eps) + (1-x) * torch.log(1-p+eps)).sum(axis=1)
print(bll1)
#tensor([-1.4271162748, -2.5879497528])

Note that to avoid log(0) error, I have introduced a very small constant eps inside it.
A better way to do this is to use BCELoss inside nn module in pytorch.
import torch.nn as nn
bce = nn.BCELoss(reduction='none')
bll2 = -bce(p, x).sum(axis=1)
print(bll2)
#tensor([-1.4271162748, -2.5879497528])

Since pytorch computes the BCE as a loss, it prepends your formula with a negative sign. The attribute reduction='none' says that I do not want the computed losses to be reduced (averaged/summed) across the batch in any way. This is advisable to use since we do not need to manually take care of numerical stability and error handling (such as adding eps above.)
You can indeed verify that the two solutions actually return the same tensor (upto a tolerance):
torch.allclose(bll1, bll2)
# True

or the tensors (without summing each row):
torch.allclose((x * torch.log(p+eps) + (1-x) * torch.log(1-p+eps)), -bce(p, x))
# True

Feel free to ask for further clarifications.


Instruction: I/O Issues in Loading Several Large H5PY Files (Pytorch)

Input:
I met a problem!
Recently I meet a problem of I/O issue. The target and input data are stored with h5py files. Each target file is 2.6GB while each input file is 10.2GB. I have 5 input datasets and 5 target datasets in total.
I created a custom dataset function for each h5py file and then use data.ConcatDataset class to link all the datasets. The custom dataset function is:
class MydataSet(Dataset):
def __init__(self, indx=1, root_path='./xxx', tar_size=128, data_aug=True, train=True):
    self.train = train
    if self.train:
        self.in_file = pth.join(root_path, 'train', 'train_noisy_%d.h5' % indx)
        self.tar_file = pth.join(root_path, 'train', 'train_clean_%d.h5' % indx)
    else:
        self.in_file = pth.join(root_path, 'test', 'test_noisy.h5')
        self.tar_file = pth.join(root_path, 'test', 'test_clean.h5')
    self.h5f_n = h5py.File(self.in_file, 'r', driver='core')
    self.h5f_c = h5py.File(self.tar_file, 'r')
    self.keys_n = list(self.h5f_n.keys())
    self.keys_c = list(self.h5f_c.keys())
    # h5f_n.close()
    # h5f_c.close()

    self.tar_size = tar_size
    self.data_aug = data_aug

def __len__(self):
    return len(self.keys_n)

def __del__(self):
    self.h5f_n.close()
    self.h5f_c.close()

def __getitem__(self, index):
    keyn = self.keys_n[index]
    keyc = self.keys_c[index]
    datan = np.array(self.h5f_n[keyn])
    datac = np.array(self.h5f_c[keyc])
    datan_tensor = torch.from_numpy(datan).unsqueeze(0)
    datac_tensor = torch.from_numpy(datac)
    if self.data_aug and np.random.randint(2, size=1)[0] == 1: # horizontal flip
        datan_tensor = torch.flip(datan_tensor,dims=[2]) # c h w
        datac_tensor = torch.flip(datac_tensor,dims=[2])

Then I use dataset_train = data.ConcatDataset([MydataSet(indx=index, train=True) for index in range(1, 6)]) for training. When only 2-3 h5py files are used, the I/O speed is normal and everything goes right. However, when 5 files are used, the training speed is gradually decreasing (5 iterations/s to 1 iterations/s). I change the num_worker and the problem still exists.
Could anyone give me a solution? Should I merge several h5py files into a bigger one? Or other methods? Thanks in advance!

Output:
Improving performance requires timing benchmarks. To do that you need to identify potential bottlenecks and associated scenarios. You said "with 2-3 files the I/O speed is normal" and "when 5 files are used, the training speed gradually decreases".  So, is your performance issue I/O speed, or training speed? Or do you know? If you don't know, you need to isolate and compare I/O performance and training performance separately for the 2 scenarios.
In other words, to measure I/O performance  (only) you need to run the following tests:

Time to read and concatenate 2-3 files,
Time to read and concatenate 5 files,
Copy the 5 files into 1, and time the read from the merged file,
Or, link the 5 files to 1 file, and time.

And to measure training speed (only) you need to compare performance for the following tests:

Merge 2-3 files, then read and train from the merged file.
Merge all 5 files, then read and train from merged file.
Or, link the 5 files to 1 file, then read and train from linked file.

As noted in my comment, merging (or linking) multiple HDF5 files into one is easy if all datasets are at the root level and all dataset names are unique.  I added the external link method because it might provide the same performance, without duplicating large data files.
Below is the code that shows both methods. Substitute your file names in the fnames list, and it should be ready to run. If your dataset names aren't unique, you will need to create unique names, and assign  in h5fr.copy() -- like  this: h5fr.copy(h5fr[ds],h5fw,'unique_dataset_name')
Code to merge -or- link files :
(comment/uncomment lines as appropriate)
import h5py
fnames = ['file_1.h5','file_2.h5','file_3.h5']
# consider changing filename to 'linked_' when using links:
with h5py.File(f'merge_{len(fnames)}.h5','w') as h5fw:
    for fname in fnames:
        with h5py.File(fname,'r') as h5fr:
            for ds in h5fr.keys():
                # To copy datasets into 1 file use:
                h5fr.copy(h5fr[ds],h5fw)
                # to link datasets to 1 file use:
                # h5fw[ds] = h5py.ExternalLink(fname,ds)


Instruction: To convert CNN model code from Keras to Pytorch

Input:
I’m trying to convert CNN model code from Keras to Pytorch.
Here is the Keras Sequential layer
model=Sequential()
model.add(Conv2D(filters=64, kernel_size = (3,3), activation="relu", input_shape=(28,28,1)))
model.add(Conv2D(filters=64, kernel_size = (3,3), activation="relu"))

model.add(MaxPooling2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Conv2D(filters=128, kernel_size = (3,3), activation="relu"))
model.add(Conv2D(filters=128, kernel_size = (3,3), activation="relu"))

model.add(MaxPooling2D(pool_size=(2,2)))
model.add(BatchNormalization())
model.add(Conv2D(filters=256, kernel_size = (3,3), activation="relu"))

model.add(MaxPooling2D(pool_size=(2,2)))

model.add(Flatten())
model.add(BatchNormalization())
model.add(Dense(512,activation="relu"))

model.add(Dense(10,activation="softmax"))

model.compile(loss="categorical_crossentropy",optimizer=optimizer,metrics=["accuracy"])

How can I initialize and write forward code on the pytorch model? Especially Flatten and Dense layer.
Any comment would appreciate.

Output:
I tried to implement it in PyTorch, but check the number of params to make sure that this is the same with your Keras implementation. I tried to write it to be more understandable and simple that's why I wrote down all activation functions. I hope this might be helpful.
import torch

import torch.nn as nn

class Net(nn.Module):
    def __init__(self, num_classes=10):
        super(Net, self).__init__()

        self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(3, 3), padding=(1, 1))
        self.relu1 = nn.ReLU(inplace=True)

        self.conv2 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=(3, 3), padding=(1, 1))
        self.relu2 = nn.ReLU(inplace=True)

        self.pool1 = nn.MaxPool2d(kernel_size=(2, 2))
        self.norm1 = nn.BatchNorm2d(num_features=64)

        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(3, 3), padding=(1, 1))
        self.relu3 = nn.ReLU(inplace=True)

        self.conv4 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=(3, 3), padding=(1, 1))
        self.relu4 = nn.ReLU(inplace=True)

        self.pool2 = nn.MaxPool2d(kernel_size=(2, 2))
        self.norm2 = nn.BatchNorm2d(num_features=128)

        self.conv5 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=(3, 3), padding=(1, 1))
        self.relu5 = nn.ReLU(inplace=True)

        self.pool3 = nn.MaxPool2d(kernel_size=(2, 2))
        self.norm3 = nn.BatchNorm2d(num_features=256)

        self.fc1 = nn.Linear(in_features=256, out_features=512)
        self.relu6 = nn.ReLU(inplace=True)

        self.fc2 = nn.Linear(in_features=512, out_features=10)
        self.act = nn.Softmax(dim=1)

    def forward(self, x):
        x = self.relu1(self.conv1(x))
        x = self.relu2(self.conv2(x))

        x = self.norm1(self.pool1(x))

        x = self.relu3(self.conv3(x))
        x = self.relu4(self.conv4(x))

        x = self.norm2(self.pool2(x))

        x = self.relu5(self.conv5(x))

        x = self.norm3(self.pool3(x))

        x = x.mean((2, 3), keepdim=True)
        x = torch.flatten(x, 1)

        x = self.relu6(self.fc1(x))
        x = self.act(self.fc2(x),)

        return x

if __name__ == '__main__':
    model = Net(num_classes=10)

    a = torch.randn(1, 3, 224, 224)

    print("Output: ", model(a).shape)
    print("Num. params: ", sum(p.numel() for p in model.parameters() if p.requires_grad))

Output
Output:  torch.Size([1, 10])
Num. params:  692938


Instruction: Number of parameters and FLOPS in ONNX and TensorRT model

Input: Does number of parameters and FLOPS (float operations per second) change when convert a model from PyTorch to ONNX or TensorRT format?

Output:
I don't think Anvar's post answered OP's question thoroughly so I did a little bit of research. Some general info before the answers to the questions as I believe OP hasn't understood fully what TensorRT and ONNX optimizations happen during the conversion from PyTorch format.
Both conversions, Pytorch to ONNX and ONNX to TensorRT increase the performance of the model by using several different optimizations. The tools actually print you information about what they do if you choose the verbose flag for them.
The preferred way to convert a Pytorch model to TensorRT is to use Torch-TensorRT as explained here.
TensorRT fuses layers and tensors in the model graph, it then uses a large kernel library to select implementations that perform best on the target GPU.
ONNX runtime offers mostly graph optimizations such as graph simplifications and node fusions to improve performance.
1. Does the number of parameters change when converting a PyTorch model to ONNX or TensorRT?
No: even though the layers are fused the number of parameters does not decrease unless there are some redundant branches in the model.
I tested this by downloading the yolov5s.onnx model here. The original model has 7.2M parameters according to the repository authors. Then I used this tool to count the number of parameters in the yolov5.onnx model and got 7225917 as a result. Thus, onnx conversion did not reduce the amount of parameters.
I was not able to get as elaborate information for TensorRT model but you can get layer information using trtexec. There is a recent question about this but there are no answers yet.
2. Does the number of FLOPS change when converting a PyTorch model to ONNX or TensorRT?
According to this post, no.


Instruction: ImportError after installing torchtext 0.11.0 with conda

Input:
I have installed pytorch version 1.10.0 alongside torchtext, torchvision and torchaudio using conda. My PyTorch is cpu-only, and I have experimented with both conda install pytorch-mutex -c pytorch and conda install pytorch cpuonly -c pytorch to install the cpuonly version, both yielding the same eror that I will describe in the following lines.
I have also installed pytorch-lightning in conda, alongside jsonargparse[summaries via pip in the environment.
I have written this code to see whether LightningCLI works or not.
# script.py
import torch
import pytorch_lightning as pl

class BoringModel(LightningModule):
    def __init__(self):
        super().__init__()
        self.layer = torch.nn.Linear(32, 2)

    def forward(self, x):
        return self.layer(x)

    def training_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("train_loss", loss)
        return {"loss": loss}

    def validation_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("valid_loss", loss)

    def test_step(self, batch, batch_idx):
        loss = self(batch).sum()
        self.log("test_loss", loss)

    def configure_optimizers(self):
        return torch.optim.SGD(self.layer.parameters(), lr=0.1)

cli = LightningCLI(BoringModel)

But when I run it using python -m script fit --print_config, I get the following error:
ImportError: /home/farhood/miniconda3/envs/pytorch_dummy_environment/lib/python3.9/site-packages/torchtext/_torchtext.so: undefined symbol: _ZNK5torch3jit6MethodclESt6vectorIN3c106IValueESaIS4_EERKSt13unordered_mapISsS4_St4hashISsESt8equal_toISsESaISt4pairIKSsS4_EEE

Which indicates that there is something broken with my Conda installation, and it's probably related to torchtext somehow.
This is the versions of the installed torch related packages:
pytorch                   1.10.0          cpu_py39hc5866cc_0    conda-forge
pytorch-lightning         1.5.2              pyhd8ed1ab_0    conda-forge
pytorch-mutex             1.0                        cuda    pytorch
torchaudio                0.10.0                 py39_cpu    pytorch
torchmetrics              0.6.0              pyhd8ed1ab_0    conda-forge
torchtext                 0.11.0                     py39    pytorch
torchvision               0.11.1                 py39_cpu    pytorch

Output:
So in order to fix the problem, I had to change my environment.yaml in order to force pytorch to install from the pytorch channel.
So this is my environment.yaml now:
channels:
  - defaults
  - pytorch
  - conda-forge
dependencies:
  # ML section
  - pytorch::pytorch
  - pytorch::torchtext
  - pytorch::torchvision
  - pytorch::torchaudio
  - pytorch::cpuonly
  - mlflow=1.21.0
  - pytorch-lightning>=1.5.2
  - pip:
    - jsonargparse[signatures]

Using this I don't get the error anymore. The pytorch related stuff installed now is:
cpuonly                   2.0                           0    pytorch
pytorch                   1.10.0              py3.9_cpu_0    pytorch
pytorch-lightning         1.5.2              pyhd8ed1ab_0    conda-forge
pytorch-mutex             1.0                         cpu    pytorch
torchaudio                0.10.0                 py39_cpu  [cpuonly]  pytorch
torchtext                 0.11.0                     py39    pytorch
torchvision               0.11.1                 py39_cpu  [cpuonly]  pytorch


Instruction: RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! When predicting with my model

Input:
I trained a model for sequence classification using transformers (BertForSequenceClassification) and I get the error:
Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! (when checking argument for argument index in method wrapper__index_select)
I don't really get where is the problem, if it's on my model, on how I tokenize the data, or what.
Here is my code:
LOADING THE PRETRAINED MODEL
model_state_dict = torch.load("../MODELOS/TRANSFORMERS/TransformersNormal",  map_location='cpu') #Doesnt work with map_location='cuda:0' neither
model = BertForSequenceClassification.from_pretrained(pretrained_model_name_or_path="bert-base-uncased", state_dict=model_state_dict, cache_dir='./data')

CREATING DATALOAD
def crearDataLoad(dfv,tokenizer):

  dft=dfv  # usamos el del validacion para que nos salga los resultados y no tener que cambiar mucho codigo

  #validation=dfv['text']
  validation=dfv['text'].str.lower()  # para modelos uncased  # el fichero que hemos llamado test es usado en la red neuronal
  validation_labels=dfv['label']

  validation_inputs = crearinputs (validation,tokenizer)
  validation_masks= crearmask (validation_inputs)

  validation_inputs = torch.tensor(validation_inputs)

  validation_labels = torch.tensor(validation_labels.values)

  validation_masks = torch.tensor(validation_masks)

  from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler# The DataLoader needs to know our batch size for training, so we specify it

  #Colab
  batch_size = 32
  #local
  #batch_size = 15

  validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)
  validation_sampler = SequentialSampler(validation_data)
  validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)

  return validation_dataloader

SHOWING RESULTS
def resultados(validation_dataloader, model, tokenizer):

  model.eval()

  # Tracking variables
  predictions , true_labels = [], []
  pred = []
  t_label =[]
  # Predict
  for batch in validation_dataloader:
    # Add batch to GPU , como no tengo lo dejo aquí
    batch = tuple(t.to(device) for t in batch)

    # Unpack the inputs from our dataloader
    b_input_ids, b_input_mask, b_labels = batch

    # Telling the model not to compute or store gradients, saving memory and
    # speeding up prediction
    with torch.no_grad():
      # Forward pass, calculate logit predictions
      outputs = model(b_input_ids, #toktype_ids=None, #
                      attention_mask=b_input_mask) #I GET THE ERROR HERE

    logits = outputs[0]

    # Move logits and labels to CPU
    logits = logits.detach().cpu().numpy()
    label_ids = b_labels.to('cpu').numpy()

    # Store predictions and true labels
    # Store predictions and true labels
    predictions.append(logits)
    true_labels.append(label_ids)

    for l in logits:
      # para cada tupla del logits, se selecciona 0 o 1 dependiendo del valor
      # que sea el mayor (argmax)
      pred_labels_i = np.argmax(l).item()
      pred.append(pred_labels_i)

  #Si no me equivoco, en pred guardamos las predicciones hechas por el modelo
  pred=np.asarray(pred).tolist()
  t_label = [val for sublist in true_labels for val in sublist] # para aplanar la lista de etiquetas
  #print('predicciones',pred)
  #print('t_labels',t_label)
  #print('validation_labels',validation_labels )
  print("RESULTADOS KFOLD validacion cruzada")
  from sklearn.metrics import confusion_matrix
  from sklearn.metrics import classification_report
  print(classification_report(t_label, pred))
  print ("Distribution test {}".format(Counter(t_label)))
  from sklearn.metrics import confusion_matrix
  print(confusion_matrix(t_label, pred))
  from sklearn.metrics import roc_auc_score
  print('AUC ROC:')
  print(roc_auc_score(t_label, pred))
  from sklearn.metrics import f1_score
  result=f1_score(t_label, pred, average='binary',labels=[0,1],pos_label=1,zero_division=0)
  print('f1-score macro:')
  print(result)
  print("****************************************************************")
  return result

I get the error at this line in function resultados:
with torch.no_grad():
     # Forward pass, calculate logit predictions
     outputs = model(b_input_ids, #toktype_ids=None, #
                     attention_mask=b_input_mask) #Esto falla

MAIN PROGRAM
trial_data = pd.DataFrame(trial_dataset)

device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
  print('no hay gpu')
print('Found GPU at: {}'.format(device_name))

#import torch# If there's a GPU available...
if torch.cuda.is_available():  # Tell PyTorch to use the GPU.
 device = torch.device("cuda")
 print('There are %d GPU(s) available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0)) # If not...
else:
 print('No GPU available, using the CPU instead.')
 device = torch.device("cpu")

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

validation_dataloader = crearDataLoad(trial_data,tokenizer)
# obteniendo metricas del modelo generado en el paso anterior
model.eval()
result= resultados(validation_dataloader, model,tokenizer)

Output: You did not move your model to device, only the data. You need to call model.to(device) before using it with data located on device.


Instruction: How to combine 2 different shaped pytorch tensors in training?

Input:
At the moment my model gives 3 output tensors. I want two of them to be more cooperative.
I want to use the combination of self.dropout1(hs) and self.dropout2(cls_hs) to pass through the self.entity_out Linear Layer. The issue is mentioned 2 tensors are in different shapes.
Current Code
class NLUModel(nn.Module):
def __init__(self, num_entity, num_intent, num_scenarios):
    super(NLUModel, self).__init__()
    self.num_entity = num_entity
    self.num_intent = num_intent
    self.num_scenario = num_scenarios

    self.bert = transformers.BertModel.from_pretrained(config.BASE_MODEL)

    self.dropout1 = nn.Dropout(0.3)
    self.dropout2 = nn.Dropout(0.3)
    self.dropout3 = nn.Dropout(0.3)

    self.entity_out = nn.Linear(768, self.num_entity)
    self.intent_out = nn.Linear(768, self.num_intent)
    self.scenario_out = nn.Linear(768, self.num_scenario)

def forward(self, ids, mask, token_type_ids):
    out = self.bert(input_ids=ids, attention_mask=mask,
                    token_type_ids=token_type_ids)

    hs, cls_hs = out['last_hidden_state'], out['pooler_output']

    entity_hs = self.dropout1(hs)
    intent_hs = self.dropout2(cls_hs)
    scenario_hs = self.dropout3(cls_hs)

    entity_hs = self.entity_out(entity_hs)
    intent_hs = self.intent_out(intent_hs)
    scenario_hs = self.scenario_out(scenario_hs)

    return entity_hs, intent_hs, scenario_hs

Required
def forward(self, ids, mask, token_type_ids):
    out = self.bert(input_ids=ids, attention_mask=mask,
                    token_type_ids=token_type_ids)

    hs, cls_hs = out['last_hidden_state'], out['pooler_output']

    entity_hs = self.dropout1(hs)
    intent_hs = self.dropout2(cls_hs)
    scenario_hs = self.dropout3(cls_hs)

    entity_hs = self.entity_out(concat(entity_hs, intent_hs)) # Concatination
    intent_hs = self.intent_out(intent_hs)
    scenario_hs = self.scenario_out(scenario_hs)

    return entity_hs, intent_hs, scenario_hs

Let's say I was successful in concatenating... will the backward propagation work?

Output:
Shape of entity_hs (last_hidden_state) is [batch_size, sequence_length, hidden_size], and shape of intent_hs (pooler_output) is just [batch_size, hidden_size] and putting them together may not make sense. It depends on what you want to do.
If, for some reason, you want to get output [batch_size, sequence_length, channels], you could tile the intent_hs tensor:
intent_hs = torch.tile(intent_hs[:, None, :], (1, sequence_lenght, 1))
... = torch.cat([entity_hs, intent_hs], dim=2)

If you want to get [batch_size, channels], you can reduce the entity_hs tensor for example by averaging:
entity_hs = torch.mean(entity_hs, dim=1)
... = torch.cat([entity_hs, intent_hs], dim=1)

Yes, the the backward pass will propagate gradients through the concatenation (and the rest).


Instruction: Pytorch index with Tensor

Input:
I have a 2-dimentional tensor arr with 0 as all the entries. I have a second tensor idx. I want to make all entries in arr  with the indices in idx into 1.
arr = torch.zeros(size = (2,10))
idx = torch.Tensor([
    [0,2],
    [4,5]
])
arr[idx] = 1 #This doesn't  work
print(arr)

The output should look like this:
tensor([[1., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 1., 1., 0., 0., 0., 0.]])

I had high confidence that I would definitely find someone else ask this in SO, however I couldn't find one.  I hope it isn't duplicate.

Output:
Use scatter() along dim=1 or the innermost dimension in this case i.e. dim=-1. Note that in place of src tensor, I just passed the constant value 1.
In [31]: arr = torch.zeros(size=(2, 10))

In [32]: idx = torch.tensor([
     ...:     [0, 2],
     ...:     [4, 5]
     ...:     ])

In [33]: torch.scatter(arr, 1, idx, 1)
Out[33]:
 tensor([[1., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
         [0., 0., 0., 0., 1., 1., 0., 0., 0., 0.]])

In [34]: torch.scatter(arr, -1, idx, 1)
Out[34]:
 tensor([[1., 0., 1., 0., 0., 0., 0., 0., 0., 0.],
         [0., 0., 0., 0., 1., 1., 0., 0., 0., 0.]])


Instruction: Can I define a method as an attribute?

Input:
The topic above is a bit ambiguous, the explaination is below:
class Trainer:
    """Object used to facilitate training."""

    def __init__(
        self,
        # params: Namespace,
        params,
        model,
        device=torch.device("cpu"),
        optimizer=None,
        scheduler=None,
        wandb_run=None,
        early_stopping: callbacks.EarlyStopping = None,
    ):
        # Set params
        self.params = params
        self.model = model
        self.device = device

        # self.optimizer = optimizer
        self.optimizer = self.get_optimizer()
        self.scheduler = scheduler
        self.wandb_run = wandb_run
        self.early_stopping = early_stopping

        # list to contain various train metrics
        # TODO: how to add more metrics? wandb log too. Maybe save to model artifacts?

        self.history = DefaultDict(list)

    @staticmethod
    def get_optimizer(
        model: models.CustomNeuralNet,
        optimizer_params: global_params.OptimizerParams(),
    ):
        """Get the optimizer for the model.

        Args:
            model (models.CustomNeuralNet): [description]
            optimizer_params (global_params.OptimizerParams): [description]

        Returns:
            [type]: [description]
        """
        return getattr(torch.optim, optimizer_params.optimizer_name)(
            model.parameters(), **optimizer_params.optimizer_params
        )

Notice that initially I passed in optimizer in the constructor, where I will be calling it outside this class. However, I now put get_optimizer inside the class itself (for consistency purpose, but unsure if it is ok). So, should I still define self.optimizer = self.get_optimizer() or just use self.get_optimizer() at the designated places in the class? The former encourages some readability for me.

Addendum: I now put the instance inside the .fit() method where I will call say 5 times to train the model 5 times. In this scenario, even though there won't be any obvious issue as we are using optimizer once per call, will it still be better to not define self.optimizer here?
    def fit(
        self,
        train_loader: torch.utils.data.DataLoader,
        valid_loader: torch.utils.data.DataLoader,
        fold: int = None,
    ):
        """[summary]

        Args:
            train_loader (torch.utils.data.DataLoader): [description]
            val_loader (torch.utils.data.DataLoader): [description]
            fold (int, optional): [description]. Defaults to None.

        Returns:
            [type]: [description]
        """
        self.optimizer = self.get_optimizer(
            model=self.model, optimizer_params=OPTIMIZER_PARAMS
        )
        self.scheduler = self.get_scheduler(
            optimizer=self.optimizer, scheduler_params=SCHEDULER_PARAMS
        )

Output: There is a difference between the two: calling your get_optimizer will instantiate a new torch.optim.<optimizer> every time. In contrast, setting self.optimizer and accessing it numerous times later will only create a single optimizer instance.


Instruction: Pytorch: RuntimeError: result type Float can't be cast to the desired output type Long

Input:
I have a model which looks as follows:
IMG_WIDTH = IMG_HEIGHT = 224

class AlexNet(nn.Module):
  def __init__(self, output_dim):
    super(AlexNet, self).__init__()
    self._to_linear = None
    self.x = torch.randn(3, IMG_WIDTH, IMG_HEIGHT).view(-1, 3, IMG_WIDTH, IMG_HEIGHT)
    self.features = nn.Sequential(
        nn.Conv2d(3, 64, 3, 2, 1), # in_channels, out_channels, kernel_size, stride, padding
        nn.MaxPool2d(2),
        nn.ReLU(inplace=True),
        nn.Conv2d(64, 192, 3, padding=1),
        nn.MaxPool2d(2),
        nn.ReLU(inplace=True),
        nn.Conv2d(192, 384, 3, padding=1),
        nn.MaxPool2d(2),
        nn.ReLU(inplace=True),
        nn.Conv2d(384, 256, 3, padding=1),
        nn.MaxPool2d(2),
        nn.ReLU(inplace=True),
        nn.Conv2d(256, 512, 3, padding=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(512, 256, 3, padding=1),
        nn.MaxPool2d(2),
        nn.ReLU(inplace=True)
  )
    self.conv(self.x)
    self.classifier = nn.Sequential(
        nn.Dropout(.5),
        nn.Linear(self._to_linear, 4096),
        nn.ReLU(inplace=True),
        nn.Dropout(.5),
        nn.Linear(4096, 4096),
        nn.ReLU(inplace=True),
        nn.Linear(4096, output_dim),
    )

  def conv(self, x):
    x = self.features(x)
    if self._to_linear is None:
        self._to_linear = x.shape[1] * x.shape[2] * x.shape[3]
    return x

  def forward(self, x):
    x = self.conv(x)
    h = x.view(x.shape[0], -1)
    x = self.classifier(h)
    return x, h

Here is my optimizer and loss functions:
optimizer = torch.optim.Adam(model.parameters())
criterion = nn.BCEWithLogitsLoss().to(device)

Here is my train and evaluate functions:
def train(model, iterator, optimizer, criterion, device):
  epoch_loss, epoch_acc = 0, 0
  model.train()
  for (x, y) in iterator:
    # features and labels to the device
    x = x.to(device)
    y = y.to(device).long()
    # Zero the gradients
    optimizer.zero_grad()
    y_pred, _ = model(x)

    # Calculate the loss and accuracy
    loss = criterion(y_pred.squeeze(), y)
    acc = binary_accuracy(y_pred, y)
    # Backward propagate
    loss.backward()
    # Update the weights
    optimizer.step()

    epoch_loss +=loss.item()
    epoch_acc += acc.item()

  return epoch_loss/len(iterator), epoch_acc/len(iterator)

def evaluate(model, iterator, criterion, device):
  epoch_loss, epoch_acc = 0, 0
  model.eval()
  with torch.no_grad():
    for (x, y) in iterator:
      x = x.to(device)
      y = y.to(device).long()
      y_pred, _ = model(x)
      loss = criterion(y_pred, y)
      acc = binary_accuracy(y_pred, y)

      epoch_loss += loss.item()
      epoch_acc += acc.item()
  return epoch_loss/len(iterator), epoch_acc/len(iterator)

This is the error that I'm getting:
RuntimeError: result type Float can't be cast to the desired output type Long

What may be possibly my problem because I have tried to convert my labels to long tensors as follows:
y = y.to(device).long()

But it seems not to work.

Output:
I was getting the same error doing this:
loss_fn(output, target)

where the output was Tensor torch.float32 and target was Tensor torch.int64. What solved this problem was calling the loss function like this:
loss_fn(output, target.float())


Instruction: Expected 4-dimensional input for 4-dimensional weight [192, 768, 1, 1], but got 2-dimensional input of size [50, 1000] instead

Input:
I'm try to modify Inception v3 pre trained in pytorch to have a multi-input. ( 4 output precisely).
I get this error:
Expected 4-dimensional input for 4-dimensional weight [192, 768, 1, 1], but got 2-dimensional input of size [50, 1000] instead
My input shape is : torch.Size([50, 3, 299, 299])
This is the code for my model,
class CNN1(nn.Module):
    def __init__(self, pretrained):
        super(CNN1, self).__init__()
        if pretrained is True:
            self.model = models.inception_v3(pretrained=True)
        modules = list(self.model.children())[:-1]      # delete the last fc layer.
        self.features = nn.Sequential(*modules)
        self.fc0 = nn.Linear(2048, 10)   #digit 0
        self.fc1 = nn.Linear(2048, 10)  #digit 1
        self.fc2 = nn.Linear(2048, 10)    #digit 2
        self.fc3 = nn.Linear(2048, 10)   #digit 3
    def forward(self, x):
        bs, _, _, _ = x.shape
        x = self.features(x)
        x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)

        label0 = self.fc0(x)
        label1 = self.fc1(x)
        label2= self.fc2(x)
        label3= self.fc3(x)

        return {'label0': label0, 'label1': label1,'label2':label2, 'label3': label3}

and this is a piece of iteration:
        for batch_idx, sample_batched in enumerate(train_dataloader):
            # importing data and moving to GPU
            image,label0, label1, label2, label3 = sample_batched['image'].to(device),\
                                             sample_batched['label0'].to(device),\
                                              sample_batched['label1'].to(device),\
                                               sample_batched['label2'].to(device) ,\
                                                       sample_batched['label3'].to(device)

            # zero the parameter gradients
            optimizer.zero_grad()
            output=model(image.float())

anyone have a suggestion?

Output:
One way to remove layers of a PyTorch model is to use nn.Identity() layer. I think you want to remove the last fully connected layer. If so check this:
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import models

class CNN1(nn.Module):
    def __init__(self, pretrained):
        super(CNN1, self).__init__()
        if pretrained is True:
            self.model = models.inception_v3(pretrained=True)
        else:
            self.model = models.inception_v3(pretrained=False)
        # modules = list(self.model.children())[:-1]
        # delete the last fc layer.
        self.model.fc = nn.Identity()

        # # to freeze training of inception weights
        # for param in self.model.parameters():
        #     param.requires_grad = False

        self.fc0 = nn.Linear(2048, 10)
        self.fc1 = nn.Linear(2048, 10)
        self.fc2 = nn.Linear(2048, 10)
        self.fc3 = nn.Linear(2048, 10)

    def forward(self, x):
        bs, _, _, _ = x.shape
        x, aux_x = self.model(x)
        # x = F.adaptive_avg_pool2d(x, 1).reshape(bs, -1)
        label0 = self.fc0(x)
        label1 = self.fc1(x)
        label2= self.fc2(x)
        label3= self.fc3(x)

        return {'label0': label0, 'label1': label1,'label2':label2, 'label3': label3}

if __name__ == '__main__':
    net = CNN1(True)
    print(net)

    inp = torch.randn(50, 3, 299, 299)
    out = net(inp)

    print('label0 shape:', out['label0'].shape)

Note: if you want to freeze training of inception layers set requires_grad = False for each parameter
In you code you are assuming all the layer connections are sequential by using nn.Sequential(*modules) line, may be that is causing the error.


Instruction: Python: Generate a unique batch from given dataset

Input:
I'm applying a CNN to classify a given dataset.
My function:
def batch_generator(dataset, input_shape = (256, 256), batch_size = 32):
    dataset_images = []
    dataset_labels = []
    for i in range(0, len(dataset)):
        dataset_images.append(cv2.resize(cv2.imread(dataset[i], cv2.IMREAD_COLOR),
                     input_shape, interpolation = cv2.INTER_AREA))
        dataset_labels.append(labels[dataset[i].split('/')[-2]])
    return dataset_images, dataset_labels

This function is supposed to be called for every epoch and it should return a unique batch of size 'batch_size' containing dataset_images (each image is 256x256) and corresponding dataset_label from the labels dictionary.
input 'dataset' contains path to all the images, so I'm opening them and resizing them to 256x256. Can someone help me in adding to this code so that is returns the desired batches?

Output:
PyTorch has two similar sounding, but very different abstractions for loading data. I strongly recommend reading the documentation on dataloaders here. To summarize

A Dataset is an object you generally implement that returns an individual sample (data + label)
A DataLoader is a built-in class in pytorch that samples batches of samples from a dataset (potentially in parallel).

A (map-style) Dataset is a simple object that just implements two mandatory methods: __getitem__ and __len__. Getitem is the method that is invoked on an object when you use the square-bracket operator i.e. dataset[i] and __len__ is the method that is invoked when you use the python built-in len function on your object, i.e. len(dataset)
For pytorch you usually want __getitem__ to return a tuple containing both the data and the label for a single item in your dataset object. For example based on what you provided, something like this should suit your needs
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms.functional as F

class CustomDataset(Dataset):
    def __init__(self, image_paths, labels, input_shape=(256, 256)):
        # `image_paths` is what you called `dataset` in your example.
        #               I'm assume this is a list of image paths.
        # `labels` isn't defined in your script but I assume its a
        #          dict that maps image names to an integer label
        #          between 0 and num classes minus 1
        self.image_paths = image_paths
        self.labels = labels
        self.input_shape = input_shape

    def __getitem__(self, index):
        # return the data and label for the specified index
        image_path = self.image_paths[index]
        data = cv2.resize(cv2.imread(image_path, cv2.IMREAD_COLOR),
                          self.input_shape, interpolation = cv2.INTER_AREA)
        label = self.labels[image_path.split('/')[-2]]

        # convert data to PyTorch tensor
        # This converts data from a uint8 np.array of shape HxWxC
        # between 0 and 255 to a pytorch float32 tensor of shape CxHxW
        # between 0.0 and 1.0.
        data = F.to_tensor(data)

        return data, label

    def __len__(self):
        return len(self.image_paths)

...
# using what you call "dataset" and "labels"
# num_workers > 0 allows you to load data in parallel while network is running
dataloader = DataLoader(
    CustomDataset(dataset, labels, (256, 256)),
    batch_size=32,
    shuffle=True,    # shuffle tells us to randomly sample the
                     # dataset without replacement
    num_workers=4    # num workers is the number of worker processes
                     # that load from dataset in parallel while your
                     # model is processing stuff
)

# training loop
for epoch in range(num_epochs):
    # iterates over all data in your dataset in a random order
    # in batches of size 32 each time this loop is run
    for data_batch, label_batch in dataloader:
        # data_batch is a pytorch FloatTensor of shape 32x3x256x256
        # label_batch is a pytorch LongTensor of shape 32

        # if using GPU acceleration now is the time to move data_batch and label_batch to GPU
        # data_batch = data_batch.cuda()
        # label_batch = label_batch.cuda()

        # zero the gradients, pass data through your model, backprop, and step the optimizer
        ...


Instruction: Weird behaviour of loss function in pytorch

Input:
I'm computing a custom cost function that is simply taking the exponential of cross-entropy divided by a parameter \eta. During the first iterations (around 20), the training loss is decreasing, but after that, I get suddenly a nan, which I don't understand why is happening.
The code I'm using is the following:
e_loss = []
eta = 2 #just an example of value of eta I'm using
criterion = nn.CrossEntropyLoss()
for e in range(epoch):
    train_loss = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        client_model.train()
        optimizer.zero_grad()
        output = client_model(data)
        loss = torch.exp(criterion(output, target)/eta) # this is the line where I input my custom loss function
        loss.backward()
        optimizer.step()
        train_loss += loss.item()*data.size(0)
    train_loss = train_loss/len(train_loader) # average losses
    e_loss.append(train_loss)

Output:
Directly using exp is quite unstable when the input is unbounded. Cross-entropy loss can return very large values if the network predicts very confidently the wrong class (b/c -log(x) goes to inf as x goes to 0). A single inaccurate prediction by your model like this can result in numerical precision that would cause gradients to go to nan, which will immediately cause weights and outputs of your model to become nan.
For example
>>> import torch
>>> import torch.nn.functional as F
>>> torch.exp(F.cross_entropy(torch.tensor([[-50.0, 50.0]]), torch.tensor([0])))
tensor(inf)


Instruction: Accumulating gradients for a larger batch size with PyTorch

Input:
In order to mimick a larger batch size, I want to be able to accumulate gradients every N batches for a model in PyTorch, like:
def train(model, optimizer, dataloader, num_epochs):
     model.train()
     model.cuda()
     for epoch_num in range(1, num_epochs+1):
         for batch_num, data in enumerate(dataloader):
             ims = data.to('cuda:0')
             loss = model(ims)
             loss.backward()
             if batch_num % N == 0 and batch_num != 0:
                 optimizer.step()
                 optimizer.zero_grad(set_to_none=True)

For this approach do I need to add the flag retain_graph=True, i.e.
loss.backward(retain_graph=True)

In this manner, are the gradients per each backward call simply summed per each parameter?

Output:
You need to set retain_graph=True if you want to make multiple backward passes over the same computational graph, making use of the intermediate results from a single forward pass. This would have been the case, for instance, if you called loss.backward() multiple times after computing loss once, or if you had multiple losses from different parts of the graph to backpropagate from (a good explanation can be found here).
In your case, for each forward pass, you backpropagate exactly once. So you don't need to store the intermediate results from the computational graph once the gradients are computed.
In short:

Intermediate outputs in the graph are cleared after a backward pass, unless explicitly preserved using retain_graph=True.
Gradients accumulate by default, unless explicitly cleared using zero_grad.


Instruction: How can I check parameters of Pytorch networks' layers?

Input:
import torch
import torch.nn as nn
from torch.optim import Adam

class NN_Network(nn.Module):
    def __init__(self,in_dim,hid,out_dim):
        super(NN_Network, self).__init__()
        self.linear1 = nn.Linear(in_dim,hid)
        self.linear2 = nn.Linear(hid,out_dim)

    def forward(self, input_array):
        h = self.linear1(input_array)
        y_pred = self.linear2(h)
        return y_pred

in_d = 5
hidn = 2
out_d = 3
net = NN_Network(in_d, hidn, out_d)

list(net.parameters())

The result was :
[Parameter containing:
 tensor([[-0.2948, -0.1261,  0.2525, -0.4162,  0.3067],
         [-0.2483, -0.3600, -0.4090,  0.0844, -0.2772]], requires_grad=True),
 Parameter containing:
 tensor([-0.2570, -0.3754], requires_grad=True),
 Parameter containing:
 tensor([[ 0.4550, -0.4577],
         [ 0.1782,  0.2454],
         [ 0.6931, -0.6003]], requires_grad=True),
 Parameter containing:
 tensor([ 0.4181, -0.2229, -0.5921], requires_grad=True)]

Without using nn.Parameter, list(net.parmeters()) results as a parameters.
What I am curious is that :

I didn't used nn.Parameter command, why does it results? And to check any network's layers' parameters, then is .parameters() only way to check it?

Maybe the result was self.linear1(in_dim,hid)'s weight, bias and so on, respectively.
But is there any way to check what it is?

Output:
Instead of .parameters(), you can use .named_parameters() to get more information about the model:
for name, param in net.named_parameters():
    if param.requires_grad:
        print(name, param.data)

Result:
linear1.weight tensor([[ 0.3727,  0.2522,  0.2381,  0.3115,  0.0656],
        [-0.3322,  0.2024,  0.1089, -0.3370,  0.3917]])
linear1.bias tensor([-0.2089,  0.1105])
linear2.weight tensor([[-0.1090,  0.2564],
        [-0.3957,  0.6632],
        [-0.4036,  0.7066]])
linear2.bias tensor([ 0.1398, -0.0585,  0.4297])


Instruction: module 'torch.optim' has no attribute 'NAdam'

Input:
nadam = torch.optim.NAdam(model.parameters())
This gives the error AttributeError: module 'torch.optim' has no attribute 'NAdam'. My PyTorch version is '1.9.1+cu102', the python version is 3.7.11. VS code does not even suggest the optimizer but the documentation clearly mentions the optimizer. I can import other optimizers like Adam

Output:
https://pytorch.org/docs/1.9.1/optim.html
From the official website, NAdam is not among the optimizers in pytorch v 1.9.1.
Try upgrading to v 1.10.0, and your code should work just fine.


Instruction: PyTorch - Computing and printing the training accuracy of the QuickStart Tutorial

Input:
to learn PyTorch, I started with the Quickstart Tutorial. In the train() method, I noticed that they don't print the training accuracy during the training session. Only the training loss is printed.
Coming from Keras, this was very unusual for me, since the training accuracy automatically printed when you call fit().
So, I decided to modify the tutorial code like the following to print the training accuracy:
def train(dataloader, model, optimizer, loss_fn):

  model.train()
  size = len(dataloader.dataset)
  num_batches = len(dataloader)

  training_loss = 0.0
  correct = 0.0

  for batch, (imgs, labels) in enumerate(dataloader):
    imgs = imgs.to(device=device)
    labels = labels.to(device=device)

    predictions = model(imgs)
    loss = loss_fn(predictions, labels)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # accumulate the training loss - each batch's loss will be added to trainin_loss
    training_loss += loss.item()

    # determines the number of correct predictions
    correct += (predictions.argmax(1) == labels).type(torch.float).sum().item()

  # end of for loop - all batches are processed

  # after all batches are processed, determine the average training loss
  training_loss = training_loss / num_batches
  # this would be the training accuracy: number of correct predictions / number of samples in dataset
  correct = correct / size

  print(f"{datetime.datetime.now()} Training Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {training_loss:>8f} \n")

Is this ok ? As a beginner to PyTorch, I wanted to make sure that is correct before I start training my neural networks.

Output:
It all looks correct. Doing things like this should not influence training. loss.backward() computes your gradients and anything not connect to that can not change them. By the way, just run the training, you can't break anything :) (Yet. Just wait when you start building self driving cars.).
I thought, in Keras/TensorFlow fit() does not compute accuracy automatically, you still have to specify this metric for example when compiling the model or as a parameter to fit(), e.g.:
model.compile(optimizer='sgd',
              loss='mse',
              metrics=[tf.keras.metrics.Accuracy()])


Instruction: How implement a Mean Standard Error (MSE) metric for NNI (Neural network intelligence) in pytorch?

Input:
I am somewhat new to pytorch since I have been using Keras for some years. Now I want to run a network architecture search (NAS) based on DARTS: Differentiable Architecture Search  (see https://nni.readthedocs.io/en/stable/NAS/DARTS.html) and it is based on pytorch.
All examples available use accuracy as a metric, but I would need to calculate MSE.
This is one of the examples available:
DartsTrainer(model,
                               loss=criterion,
                               metrics=lambda output, target: accuracy(output, target, topk=(1,)),
                               optimizer=optim,
                               num_epochs=args.epochs,
                               dataset_train=dataset_train,
                               dataset_valid=dataset_valid,
                               batch_size=args.batch_size,
                               log_frequency=args.log_frequency,
                               unrolled=args.unrolled,
                               callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("./checkpoints")])

# where the accuracy is defined in a separate function:

def accuracy(output, target, topk=(1,)):
    # Computes the precision@k for the specified values of k
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    # one-hot case
    if target.ndimension() > 1:
        target = target.max(1)[1]

    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = dict()
    for k in topk:
        correct_k = correct[:k].reshape(-1).float().sum(0)
        res["acc{}".format(k)] = correct_k.mul_(1.0 / batch_size).item()
    return res

As I see in pytorch it is more complicated to calculate metrics then in Keras. Can someone help please?
As a trial, I wrote this code:
def accuracy_mse(output, target):
    batch_size = target.size(0)

    diff = torch.square(output.t()-target)/batch_size
    diff = diff.sum()

    res = dict()

    res["acc_mse"] = diff
    return res

It seems to be working, but I am not 100% sure about it ...

Output:
Finally I figured out that the transpose (.t() ) wac causing the problem, so the final code is:
def accuracy_mse(output, target):

    """ Computes the mse """
    batch_size = target.size(0)

    diff = torch.square(output-target)/batch_size
    diff = diff.sum()
    res = dict()

    res["mse"] = diff

    return res


Instruction: PyTorch equivalent to Tensorflow's tf.keras.dot() in terms of behaviour?

Input:
In tensorflow, if you have 2 tensors of shape NxTxD and NxDxT respectively (N=batch_size, T=SequenceLength, D=NumberOfFeatures), you can dot them and get an output of NxTxT, as demonstrated below:
import tensorflow as tf
import numpy as np

x1 = np.arange(2 * 4 * 3).reshape(2, 4, 3)
x2 = np.flip(np.arange(2 * 4 * 3).reshape(2, 3, 4), 1).copy()
print(x1.shape, x2.shape)
dotted = tf.keras.layers.Dot(axes=(2, 1))([x1, x2])
print(dotted.shape)
dotted

(2, 4, 3) (2, 3, 4)
(2, 4, 4)
<tf.Tensor: shape=(2, 4, 4), dtype=int32, numpy=
array([[[   4,    7,   10,   13],
        [  40,   52,   64,   76],
        [  76,   97,  118,  139],
        [ 112,  142,  172,  202]],

       [[ 616,  655,  694,  733],
        [ 760,  808,  856,  904],
        [ 904,  961, 1018, 1075],
        [1048, 1114, 1180, 1246]]])>

If you try to do the same in PyTorch, the result is different:
import torch
import numpy as np

x1 = torch.from_numpy(np.arange(2 * 4 * 3).reshape(2, 4, 3))
x2 = torch.from_numpy(np.flip(np.arange(2 * 4 * 3).reshape(2, 3, 4), 1).copy())
dotted = torch.tensordot(x1, x2, dims=([2], [1]))
print(x1.shape, x2.shape)
print(dotted.shape)
dotted

torch.Size([2, 4, 3]) torch.Size([2, 3, 4])
torch.Size([2, 4, 2, 4])
tensor([[[[   4,    7,   10,   13],
          [  40,   43,   46,   49]],

         [[  40,   52,   64,   76],
          [ 184,  196,  208,  220]],

         [[  76,   97,  118,  139],
          [ 328,  349,  370,  391]],

         [[ 112,  142,  172,  202],
          [ 472,  502,  532,  562]]],

        [[[ 148,  187,  226,  265],
          [ 616,  655,  694,  733]],

         [[ 184,  232,  280,  328],
          [ 760,  808,  856,  904]],

         [[ 220,  277,  334,  391],
          [ 904,  961, 1018, 1075]],

         [[ 256,  322,  388,  454],
          [1048, 1114, 1180, 1246]]]], dtype=torch.int32)

Now, Tensorflow's results exist inside the results that pytorch produces (it's a subset of it).
In fact, tensorflow's results is basically some kind of "diagonal" in higher dimensions.
PyTorch's output is NxTxNxT, so to get exactly the same results as Tensorflow you can do:
torch.stack([dotted[i, :, i, :] for i in range(len(dotted))])

tensor([[[   4,    7,   10,   13],
         [  40,   52,   64,   76],
         [  76,   97,  118,  139],
         [ 112,  142,  172,  202]],

        [[ 616,  655,  694,  733],
         [ 760,  808,  856,  904],
         [ 904,  961, 1018, 1075],
         [1048, 1114, 1180, 1246]]], dtype=torch.int32)

but this doesn't negate the fact that you're both:

Allocating memory for a tensor of NxTxNxT instead of NxTxT
The computational complexity/time increases dramatically

Is there a way to get the same 3 dimensional results that tensorflow gives from pytorch, without it computing the 4 dimensional tensor?

Output: I hope you are looking for batch matrix multiplication (bmm) which multiplies two batches of matrices - the two tensors have to be 3D. https://pytorch.org/docs/stable/generated/torch.bmm.html


Instruction: Create custom convolutional Loss function that only takes parts of the tensor

Input:
I have a convolutional network that gets images, but also a colored border on each image for additional information input to the network. Now I want to calculate the loss, but the usual loss function will also take the predicted border into account. The border is completely random, and is just an input to the system. I don’t want the model to think it has performed badly, when it predicted the wrong color.
This happens in the DataLoader.getitem:
def __getitem__(self, index):
        path = self.input_data[index]
        imgs_path = sorted(glob.glob(path + '/*.png'))
        #read light conditions
        lightConditions = []
        with open(path +"/lightConditions.json", 'r') as file:
            lightConditions = json.load(file)
        #shift light conditions
        lightConditions.pop(0)
        lightConditions.append(False)
        frameNumber = 0
        imgs = []
        for img_path in imgs_path:
            img = cv2.imread(img_path)
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
            im_pil = Image.fromarray(img)
            #img = cv2.resize(img, (256,448))
            if lightConditions[frameNumber] ==False:
                imgBorder = ImageOps.expand(im_pil,border = 6, fill='black')
            else:
                imgBorder = ImageOps.expand(im_pil, border = 6, fill='orange')
            img = np.asarray(imgBorder)
            img = cv2.resize(img, (256,448))
            #img = cv2.resize(img, (0, 0), fx=0.5, fy=0.5, interpolation=cv2.INTER_CUBIC) #has been 0.5 for official data, new is fx = 2.63 and fy = 2.84

            img_tensor = ToTensor()(img).float()
            imgs.append(img_tensor)
            frameNumber +=1
        imgs = torch.stack(imgs, dim=0)
        return imgs

And then this is done in training:
for idx_epoch in range(startEpoch, nEpochs):

        #set epoch in dataloader for right shuffle ->set seed really random
        val_loader.sampler.set_epoch(idx_epoch)
        #Remember time for displaying time for epoch
        startTimeEpoch = datetime.now()
        i = 0
        if processGPU==0:
            running_loss = 0
        beenValuated = False
        for index, data_sr in enumerate(train_loader):
            #Transfer Data to GPU but don't block other processes because this only effects this single process
            data_sr = data_sr.cuda(processGPU, non_blocking=True)

            startTimeIteration = time.time()
            #Remove all dimensions of size 1
            data_sr = data_sr.squeeze()
            # calculate the index of the input images and GT images
            num_f = len(data_sr)
            #If model_type is 0 -> only calculate one frame that is marked with gt
            if cfg.model_type == 0:
                idx_start = random.randint(-2, 2)
                idx_all = list(np.arange(idx_start, idx_start + num_f).clip(0, num_f - 1))
                idx_gt = [idx_all.pop(int(num_f / 2))]
                idx_input = idx_all
            #Else when model_type is 1 then input frames 1,2,3 and predict frame 4 to number of cfg.dec_frames. Set all images that will be predicted to 'gt' images
            else:
                idx_all = np.arange(0, num_f)
                idx_input = list(idx_all[0:4])
                idx_gt = list(idx_all[4:4+cfg.dec_frames])
            imgs_input = data_sr[idx_input]
            imgs_gt = data_sr[idx_gt]

            # get predicted result
            imgs_pred = model(imgs_input)

I use cfg.model_type = 1. This model will give me new images with also a colored border. And usually here follows a loss calculation:
loss = criterion_mse(imgs_pred, imgs_gt)

But I can no longer use this. Does anyone know how to write a custom loss function that only takes certain parts of the tensor into account or which parts in the tensor represent which images?

Output:
You can slice tensors the same way as in numpy. The dimensions of image batches are NCHW. If b is your border size and it is symmetric from all sides then just crop the tensors:
loss = criterion_mse(imgs_pred[:, :, b:-b, b:-b] , imgs_gt[:, :, b:-b, b:-b])


Instruction: Policy Network returning different outputs for batched states and individual states

Input:
I am implementing REINFORCE applied to the CartPole-V0 openAI gym environment.  I am trying 2 different implementations of the same, and the issue I am not able to resolve is the following:
Upon passing a single state to the Policy Network, I get an output Tensor of size 2, containing the action probabilities of the 2 actions. However, when I pass a `batch of states' to the Policy Network to compute the output action probabilities of all of them, the values that I obtain are very different from when each state is individually passed to the network.
Can someone help me understand the issue?
My code for the same is below: (Note: this is NOT the complete REINFORCE algorithm -- I am aware that I need to compute the loss from the probabilities. But I am trying to understand the difference in the computation of the two probabilities, which I think should be the same, before proceeding.)
# architecture of the Policy Network
class PolicyNetwork(nn.Module):
    def __init__(self, state_dim, n_actions):
        super().__init__()
        self.n_actions = n_actions
        self.model = nn.Sequential(
            nn.Linear(state_dim, 64),
            nn.ReLU(),
            nn.Linear(64, n_actions),
            nn.Softmax(dim=0)
        ).float()

    def forward(self, X):
        return self.model(X)

def train_reinforce_agent(env, episode_length, max_episodes, gamma, visualize_step, learning_rate=0.003):

    # define the parametric model for the Policy: this is an instantiation of the PolicyNetwork class
    model = PolicyNetwork(env.observation_space.shape[0], env.action_space.n)
    # define the optimizer for updating the weights of the Policy Network
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    # hyperparameters of the reinforce agent
    EPISODE_LENGTH = episode_length
    MAX_EPISODES = max_episodes
    GAMMA = gamma
    VISUALIZE_STEP = max(1, visualize_step)
    score = []

    for episode in range(MAX_EPISODES):
        # reset the environment
        curr_state = env.reset()
        done = False
        episode_t = []

        # rollout an entire episode from the Policy Network
        pred_vals = []
        for t in range(EPISODE_LENGTH):
            act_prob = model(torch.from_numpy(curr_state).float())
            pred_vals.append(act_prob)
            action = np.random.choice(np.array(list(range(env.action_space.n))), p=act_prob.data.numpy())
            prev_state = curr_state
            curr_state, _, done, info = env.step(action)
            episode_t.append((prev_state, action, t+1))
            if done:
                break
        score.append(len(episode_t))
        # reward_batch = torch.Tensor([r for (s,a,r) in episode_t]).flip(dims=(0,))
        reward_batch = torch.Tensor([r for (s, a, r) in episode_t])

        # compute the return for every state-action pair from the rewards at every time-step
        batch_Gvals = []
        for i in range(len(episode_t)):
            new_Gval = 0
            power = 0
            for j in range(i, len(episode_t)):
                new_Gval = new_Gval + ((GAMMA ** power) * reward_batch[j]).numpy()
            power += 1
            batch_Gvals.append(new_Gval)

        # normalize the returns for the batch
        expected_returns_batch = torch.FloatTensor(batch_Gvals)
        if torch.is_nonzero(expected_returns_batch.max()):
            expected_returns_batch /= expected_returns_batch.max()

        # batch the states, actions, prob after the episode
        state_batch = torch.Tensor([s for (s,a,r) in episode_t])
        print("State batch:", state_batch)
        all_states = [s for (s,a,r) in episode_t]
        print("All states:", all_states)
        action_batch = torch.Tensor([a for (s,a,r) in episode_t])
        pred_batch_v1 = model(state_batch)
        pred_batch_v2 = torch.stack(pred_vals)
        print("Batched state pred_vals:", pred_batch_v1)
        print("Individual state pred_vals:", pred_batch_v2) ### Why is this different from the above predicted values??

My main function where I pass the environment is:
def main():
    env = gym.make('CartPole-v0')
    # train a REINFORCE-agent to learn the optimal policy
    episode_length = 500
    n_episodes = 500
    gamma = 0.99
    vis_steps = 50
    train_reinforce_agent(env, episode_length, n_episodes, gamma, vis_steps)

Output: In your policy, you have Softmax over dim 0. This normalizes the probability of each action across your batch. You want to do it across actions by dim=1.


Instruction: How to minimize wrt one set of parameters and maximize wrt other set of parameters simultaneously in a training loop in pytorch?

Input:
I have a loss function that includes two sets of parameters to learn. One is a matrix, wrt which I want to maximize the loss, other is the set of parameters for logistic regression, wrt which I want to minimize the loss.
In pytorch whenever I use loss.backward(), the loss is minimized wrt both sets of parameters and (-loss).backward() maximizes wrt both. How do I do minimax optimization wrt the sets of parameters in pytorch?
Tensorflow probably has this concept of gradient_tape and tape.watch() concept. What's the alternative in pytorch?

Output:
You can refer to the gradient reversal idea from https://arxiv.org/abs/1409.7495.
But the crux of the idea is this: you have some loss function l(X,Y) where X and Y are parameters. Now you want to update X to minimize loss and update Y to maximize loss, which can be seen as minimizing -l(X,Y).
Essentially you want to update parameters X with dl/dX and Y with d(-l)/dY = -dl/dy. You can do this by doing a backpropagation step, modifying the gradients of Y, and applying the update. In pytorch terms, that would be:
loss = compute_loss()
loss.backward()
# modify gradients of Y
Y.grad.data = -Y.grad.data
optimizer.step()
optimizer.zero_grad()