Instruction: Why my CNN regressor doesn't work (Pytorch)

Input:
I'm trying to convert my tensorflow code to pytorch.
Simply speaking, it estimates 7 values (number) from images using CNN.(regressor)
The backbone network is vgg16 with pretrained weights, I'd like to convert last fcl (actually  due to ImageNet dataset, the last fcl output is 1000 classes), to (4096 x 4096), and add more fcls.
before :
vgg last fcl (4096 x 1000)
after:
vgg last fcl (change to 4096 x 4096)
----add fcl1 (4096 x 4096)
----add fcl2 (4096 x 2048)
└ add fclx (2048 x 3)
└ add fclq (2048 x 4)
: fcl2 is connected to two different tensors, with size of 3 and 4
Here, I tried to do it with only one image (for just debugging) and GT values (7 values) with L2 Loss.
If I do that using Tensorflow, the loss decreases drastically, and When I Infer an image, it gives almost similar values to GT.
However, If I try to do it using Pytorch, It looks like training doesn't work well.
I guess the loss should sharply decrease while training (almost for every iteration)
What's the problem?

The loss is actually |x-x'|^2 + b|q-q'|^2, well-known as L2-norm used in PoseNet(Kendall, 2015). x has three values of position and q has four values of quaternion(rotation). b is the hyperparameter determined by user.

from torchvision import models
import torch.nn as nn
import torch
from torch.autograd import Variable
import torch.optim as optim
import os
import os.path
import torch.utils.data as data
from torchvision import transforms as T
from PIL import Image

class DataSource(data.Dataset):
    def __init__(self, root, train=True, transforms=None, txtName='dataset_train'):
        self.root = os.path.expanduser(root)
        self.transforms = transforms
        self.train = train
        self.imageFormat = '.jpg'
        self.image_poses = []
        self.image_paths = []
        self.txtName = txtName
        self._get_data()

        if transforms is None:
            normalize = T.Normalize(mean=[0.485, 0.456, 0.406],
                                    std=[0.229, 0.224, 0.225])
            if not train:
                self.transforms = T.Compose(
                    [T.Resize(256),
                     T.CenterCrop(224),
                     T.ToTensor(),
                     normalize]
                )
            else:
                self.transforms = T.Compose(
                    [T.Resize(256),
                     T.CenterCrop(224),
                     # T.RandomCrop(224),
                     T.ToTensor(),
                     normalize]
                )

    def _get_data(self):
        txt_file = self.root + '/' + self.txtName + '.txt'
        count = 0
        with open(txt_file, 'r') as f:
            for line in f:
                if len(line.split()) != 8:
                    next(f)
                fname, p0, p1, p2, p3, p4, p5, p6 = line.split()
                p0 = float(p0); p1 = float(p1); p2 = float(p2);
                p3 = float(p3); p4 = float(p4); p5 = float(p5); p6 = float(p6)
                ImageFullName = self.root + '/' +  fname
                if count == 0:
                    if os.path.isfile(ImageFullName) == False:
                        self.imageFormat = '.png'

                if self.imageFormat != '.jpg':
                    ImageFullName = ImageFullName.replace('.jpg', self.imageFormat)

                self.image_poses.append([p0, p1, p2, p3, p4, p5, p6])
                self.image_paths.append(ImageFullName)
                count += 1
        print('Total : ', len(self.image_paths), ' images')

    def __getitem__(self, index):
        img_path = self.image_paths[index]
        img_pose = self.image_poses[index]
        data = Image.open(img_path)
        data = self.transforms(data)
        return data, torch.tensor(img_pose)

    def __len__(self):
        return len(self.image_paths)

class PoseLoss(nn.Module):
    def __init__(self, beta, device = 'cuda'):
        super(PoseLoss, self).__init__()
        self.beta = beta
        self.device = device
        self.t_loss_fn = nn.MSELoss()

    def forward(self, x, q, poseGT):
        GT_x = poseGT[:, 0:3]
        GT_q = poseGT[:, 3:]

        xx = Variable(x, requires_grad=True).to(self.device)
        qq = Variable(q, requires_grad=True).to(self.device)
        print('GT', GT_x, GT_q)
        print('Estim', xx, qq)

        loss = torch.sqrt(self.t_loss_fn(GT_x[:, :3].cpu(), xx[:, :3].cpu())) + self.beta*torch.sqrt(self.t_loss_fn(GT_q[:, 3:].cpu(), qq[:, 3:].cpu()))
        return loss

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.backbone =  models.vgg16(pretrained=True)
        self.backbone._modules['classifier'][6] = nn.ReLU(nn.Linear(4096, 4096))
        self.fcl = nn.Sequential(nn.Linear(4096, 4096), nn.ReLU(), nn.Linear(4096, 2048), nn.ReLU())
        self.xyz = nn.Linear(2048, 3)
        self.q = nn.Linear(2048, 4)

    def forward(self, x):
        x1 = self.backbone(x)
        x2 = self.fcl(x1)
        xyz = self.xyz(x2)
        q = self.q(x2)
        return xyz, q

batch_size = 1
learning_rate = 10e-5
training_epochs = 100

if __name__ == "__main__":
    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    data = DataSource(DatasetDirectory + DatasetFolder, train=True, transforms=None, txtName=TrainDatasetList)
    data_loader = torch.utils.data.DataLoader(dataset=data, batch_size=batch_size, shuffle=False, num_workers=4)

    model = Net().to(device)

    model.train()

    criterion = PoseLoss(beta = 100, device = device)

    optimizer = optim.Adam(model.parameters(), lr=learning_rate, betas = (0.9, 0.999), eps =0.00000001)

    iteration = 0
    minloss = 10e8
    minlossindex = -1
    for epoch in range(1, training_epochs):
        dataiter = iter(data_loader)
        for Images, Poses in dataiter:
            optimizer.zero_grad()
            Images = Images.to(device).float()
            x, q = model(Images)
            loss = criterion(x, q, Poses)
            loss.backward()
            loss = loss.item()/ batch_size
            optimizer.step()
            print(epoch, ' : ', iteration , ' -> ' , loss, ' minloss ', minloss, ' at ', minlossindex)
            if loss < minloss:
                minloss = loss
                minlossindex = iteration
                if epoch < (int)(training_epochs*0.8):
                    torch.save(model.state_dict(), 'Min.pth')
            iteration = iteration + 1
    torch.save(model.state_dict(), 'Fin.pth')

The estimated results tends to be zero for all 7 values, I cannot come up with why it gives such values.

Also, as I mentioned above, the loss values do not decrease dramatically while training(I expected It should be decreased dramatically for every iteration until it converges, because I used only one image for training)

Output:
If you move the data from gpu to cpu, you lose the historic in the computation graph and therefore the derivative is not propagated to the previous layers.
I do the following, usually the data is transferred to the device after sampling with the dataloader.
...
for Images, Poses in dataiter:
    Images = Images.to(device)
    Poses = Poses.to(device)
...

From here you will have all the data in gpu. Also, is not necessary to apply the variable in x and q. Automatically, when a layer is defined in pytorch it is already indicated that the tensor is a variable and that it must have an accumulation of gradient.
On the other hand, you don't need the sqrt in the loss either. Think that sqrt function is monomically increasing, so minimize mse is the same as minimize rmse. Putting the sqrt will probably make the training somewhat more unstable and will only be useful if you want to penalize in the same order of magnitude as the data.


Instruction: Define following multiplication of two tensors in pytorch lightning

Input:
I would like to to multiply following two tensors x (of shape (BS, N, C)) and y (of shape (BS,1,C)) in the following way:
BS = x.shape[0]
N = x.shape[1]
out = torch.zeros(size=x.shape)
for i in range(BS):
    for j in range(N):
        out[i, j, :] = torch.mul(x[i, j, :], y[i, 0, :])
return out

Implementing it this way yields an error "RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! (when checking arugment for argument weight in method wrapper_native_layer_norm)"

When fixing it by setting
out = torch.zeros(size=x.shape).to('cuda)

then training takes forever, because my for loops aren't executed in parallel.
So my question is how to implement the two for loops above in the pytorch-lightning way, so that I can define function x = multiply_as_above(x, y) and use it in the feedword(self) method of my neural network.
Btw the operation defined above looks to me like a convolution with kernel size 1. Maybe I can use that?

Output:
Is there anything wrong with x*y? As you can see in the code below, it yields exactly the same output as your function:
import torch

torch.manual_seed(2021)

BS = 2
N = 3
C = 4

x = torch.rand(BS, N, C)
y = torch.rand(BS, 1, C)

# your function
def f(x, y):
    BS = x.shape[0]
    N = x.shape[1]
    out = torch.zeros(size=x.shape)
    for i in range(BS):
        for j in range(N):
            out[i, j, :] = torch.mul(x[i, j, :], y[i, 0, :])
    return out

out1 = f(x, y)
out2 = x*y

# comparing the outputs, we can see that they are identical
torch.all(out1 == out2)
# > tensor(True)


Instruction: Delete a row by index from pytorch tensor

Input:
I have a pytorch tensor of size torch.Size([4, 3, 2])
tensor([[[0.4003, 0.2742],
     [0.9414, 0.1222],
     [0.9624, 0.3063]],

    [[0.9600, 0.5381],
     [0.5758, 0.8458],
     [0.6342, 0.5872]],

    [[0.5891, 0.9453],
     [0.8859, 0.6552],
     [0.5120, 0.5384]],

    [[0.3017, 0.9407],
     [0.4887, 0.8097],
     [0.9454, 0.6027]]])

I would like to delete the 2nd row so that the tensor becomes torch.Size([3, 3, 2])
tensor([[[0.4003, 0.2742],
     [0.9414, 0.1222],
     [0.9624, 0.3063]],

    [[0.5891, 0.9453],
     [0.8859, 0.6552],
     [0.5120, 0.5384]],

    [[0.3017, 0.9407],
     [0.4887, 0.8097],
     [0.9454, 0.6027]]])

How can I delete the nth row of the 3D tensor?

Output:
import torch
x = torch.randn(size=(4,3,2))

row_exclude = 2
x = torch.cat((x[:row_exclude],x[row_exclude+1:]))

print(x.shape)
>>> torch.Size([3, 3, 2])


Instruction: train a model which is instantiated in another model ( Pytorch)

Input:
I have two classes of networks of neurons one of GNN type and the other simple of linear type, the latter is instantiated in the first !!! how can I train both at the same time?
here is an example:
class linear_NN(nn.Module):

  def __init__(self, input_dim, out_dim...):
    super().__init__()

  def forward(self, x, dim = 0):
    '''Forward pass'''
    return x

the main class or the large class
class GNN(nn.Module):

  def __init__(self, input_dim, n-hidden, out_dim...):
    super().__init__()

  def forward(self, h, dim = 0):
    '''Forward pass'''
    model=linear_NN(input, out..)
    model(h, dim)
    return h

Output:
You must declare it in the __init__(...):
class GNN(nn.Module):
  def __init__(self, input_dim, n-hidden, out_dim, ...):
    super().__init__()
    self.linear = linear_NN(input, out..)

  def forward(self, h, dim = 0):
    '''Forward pass'''
    self.linear(h, dim)
    return h

Then, the self.linear model will be registered to your GNN main model, and if you get GNN(...).parameters(), you'll see the linear parameters there.


Instruction: 'torchmetrics' does not work with PyTorchLightning

Input:
I am trying to understand how to use torchmetrics with PyTorch Lightning.
But, I got a same output with Accuracy, F1-score, Precision, etc.
This is the code.

metric_acc = torchmetrics.Accuracy()
metric_f1 = torchmetrics.F1()
metric_pre = torchmetrics.Precision()
metric_rec = torchmetrics.Recall()

n_batches = 3
for i in range(n_batches):
    # simulate a classification problem
    preds = torch.randn(10, 5).softmax(dim=-1)
    target = torch.randint(5, (10,))

    acc = metric_acc(preds, target)
    f1 = metric_f1(preds, target)
    pre = metric_pre(preds, target)
    rec = metric_rec(preds, target)
    print(f"Accuracy on batch {i}: {acc}")
    print(f"F1 score on batch {i}: {f1}")
    print(f"pre score on batch {i}: {pre}")
    print(f"rec score on batch {i}: {rec}")
    print('-' * 20)

acc = metric_acc.compute()
f1 = metric_f1.compute()
pre = metric_pre.compute()
rec = metric_rec.compute()
print(f"Accuracy on all data: {acc}")
print(f"f1 score on all data: {f1}")
print(f"pre score on all data: {pre}")
print(f"rec score on all data: {rec}")

Result is here.

Accuracy on batch 0: 0.10000000149011612
F1 score on batch 0: 0.10000000894069672
pre score on batch 0: 0.10000000149011612
rec score on batch 0: 0.10000000149011612
--------------------
Accuracy on batch 1: 0.30000001192092896
F1 score on batch 1: 0.30000001192092896
pre score on batch 1: 0.30000001192092896
rec score on batch 1: 0.30000001192092896
--------------------
Accuracy on batch 2: 0.4000000059604645
F1 score on batch 2: 0.40000003576278687
pre score on batch 2: 0.4000000059604645
rec score on batch 2: 0.4000000059604645
--------------------
Accuracy on all data: 0.2666666805744171
f1 score on all data: 0.2666666805744171
pre score on all data: 0.2666666805744171
rec score on all data: 0.2666666805744171

Process finished with exit code 0

I got the same result when I used it with PyTorchLightning, so I try it with simple code and got the same thing.If you know the problem or the solution, please let me know.Thank you very much.

Output:
The reason for this is that for multi class classification if you are using F1, Precision, ACC and Recall with micro (the default )these are equivalent metrics and recommending you should use macro
metric_acc = torchmetrics.Accuracy(average='macro')
metric_f1 = torchmetrics.F1(average='macro')
metric_pre = torchmetrics.Precision(average='macro')
metric_rec = torchmetrics.Recall(average='macro')


Instruction: Share the output of one class to another class python

Input:
I have two DNNs the first one returns two outputs. I want to use one of these outputs in a second class that represents another DNN as in the following example:
I want to pass the output (x) to the second class to be concatenated to another variable (v). I found a solution to make the variable (x) as a global variable, but I need another efficient solution
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

class Net(nn.Module):
    def __init__(self):
        ..
    def forward(self, x):
        ..
        return x, z

class Net2(nn.Module):
    def __init__(self):
        ..
    def forward(self, v):
        y = torch.cat(v, x)
        return y

Output:
You should not have to rely on global variables, you need to solve this following common practices. You can pass both v, and x as parameters of the forward of Net2. Something like:
class Net(nn.Module):
    def forward(self, x):
        z = x**2
        return x, z

class Net2(nn.Module):
    def forward(self, x, v):
        y = torch.cat((v, x), dim=1)
        return y

With dummy data:
>>> net = Net()
>>> net2 = Net2()

>>> input1 = torch.rand(1,10)
>>> input2 = torch.rand(1,20)

First inference:
>>> x, z = net(input1)

Second inference:
>>> out = net2(x, input2)
>>> out.shape
torch.Size([1, 30])


Instruction: Dataset with 4D images: expected Byte but found Float

Input:
I have some MRI scans that I want to create a custom PyTorch Dataset out of. Each scan is a set of 31 RGB images, so the scans are 4 dimensional (Channels, Depth, Height, Width). The images are .png, and each scan is a folder with 31 images. After loading the scans, I tried passing them through a Conv3D, but I got an error (full traceback at the end):
x = torch.unsqueeze(dataset[0][0], 0)
x.shape  # torch.Size([1, 3, 31, 512, 512])
m = nn.Conv3d(3,12,3)
out = m(x)

RuntimeError: expected scalar type Byte but found Float

How can I solve this error? I think it happens because I load the scans in as a NumPy array of NumPy arrays, but I don't know how else to do it. How can I load 4D image data into a custom Dataset?
Here's my custom Dataset class:
import torch
import os
import pandas as pd
from skimage import io
from torch.utils.data import Dataset

class TrainImages(Dataset):
    def __init__(self, csv_file, root_dir, transform=None):
        self.annotations = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.annotations)

    def __getitem__(self, index):
        # The folder containing the images of a scan
        img_path = os.path.join(self.root_dir, str(self.annotations.iloc[index, 0]).zfill(5))
        # Create a tensor out of a numpy array of numpy arrays, where each array is an image in the scan
        image = torch.from_numpy(np.array([np.array(Image.open(os.path.join(str(img_path),"rgb-"+str(i)+".png"))) for i in range(31)]).transpose(3,0,1,2).astype(np.uint8))
        y_label = torch.tensor(int(self.annotations.iloc[index, 1]))
        return (image, y_label)

Full traceback:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-29-f3c4dfbd5496> in <module>
      1 m=nn.Conv3d(3,12,3)
----> 2 out=m(x)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    571                             self.dilation, self.groups)
    572         return F.conv3d(input, self.weight, self.bias, self.stride,
--> 573                         self.padding, self.dilation, self.groups)
    574
    575

RuntimeError: expected scalar type Byte but found Float

Output:
The error message can be confusing, but the problem is that your data has the Byte type, while conv3d expects Float. You need to change from np.uint8 to np.float32 in the __getitem__(...) of your Dataset:
image = torch.from_numpy(np.array([
    np.array(Image.open(os.path.join(str(img_path),"rgb-"+str(i)+".png")))
    for i in range(31)
]).transpose(3, 0, 1, 2).astype(np.float32))  # <<< changed from np.uint8 to float32

or, cast x to Float before passing to the model:
out = m(x.float())

Note that if you use a transform like .ToTensor() later on, this problem will be solved as well.


Instruction: Pytorch: non-positive stride is not supported

Input:
I have some MRI scans, where each scan is a set of 31 RGB images. The dimensions of the input data are (Channels, Depth, Height, Width). The images are png, and each scan is a folder containing its 31 images.
I created a custom Dataset class:
class TrainImages(Dataset):
    def __init__(self, csv_file, root_dir, transform=None):
        self.annotations = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.annotations)

    def __getitem__(self, index):
        img_path = os.path.join(self.root_dir, str(self.annotations.iloc[index, 0]).zfill(5))
        image = torch.from_numpy(np.array([np.array(Image.open(os.path.join(str(img_path),"rgb-"+str(i)+".png"))) for i in range(31)]).transpose(3,0,1,2).astype(np.float32))
        y_label = torch.tensor(int(self.annotations.iloc[index, 1]))

        return (image, y_label)

Then, I created a small 3D CNN class:
class CNN2(nn.Module):
    def __init__(self):
        super(CNN2, self).__init__()
        self.conv_layer1 = self._conv_layer(3, 12)

    def _conv_layer(self, in_c, out_c, conv_kernel_size=3, padding=0):
        layer = nn.Sequential(
        nn.Conv3d(in_c, out_c, conv_kernel_size, padding),
        )

        return layer

    def forward(self, x):
        out = self.conv_layer1(x)

        return out

Then, I tried to feed one scan into the CNN2 object:
    x=torch.unsqueeze(dataset[0][0], 0)
    x.shape #torch.Size([1, 3, 31, 512, 512])
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    data = x.to(device)
    model = CNN2().to(device)
    model(x)

But it produces this error:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-24-c89306854a22> in <module>
      1 model = CNN_test().to(device)
----> 2 model(x)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

<ipython-input-22-ac66ca7a2459> in forward(self, x)
     14
     15     def forward(self, x):
---> 16         out = self.conv_layer1(x)
     17
     18         return out

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    115     def forward(self, input):
    116         for module in self:
--> 117             input = module(input)
    118         return input
    119

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    571                             self.dilation, self.groups)
    572         return F.conv3d(input, self.weight, self.bias, self.stride,
--> 573                         self.padding, self.dilation, self.groups)
    574
    575

RuntimeError: non-positive stride is not supported

However, when I just create a Conv3D object and pass the same scan in, no error results:
x=torch.unsqueeze(dataset[0][0], 0)
m=nn.Conv3d(3,12,3)
out=m(x)

I think the error might have to do with the dimensions of the input data, but I don't understand what "non-positive stride" means. I'm also confused why no error occurs when I just pass the data into a Conv3D object, but an error occurs when I pass the same data into an instance of the CNN class that does the same thing.

Output:
The issue is not with your input shape, it has to do with your layer initialization. You have essentially defined your 3D convolution with this line:
nn.Conv3d(in_c, out_c, conv_kernel_size, padding)

The issue is nn.Conv3d function head is the following:

torch.nn.Conv3d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros', device=None, dtype=None)

Notice how stride is placed before padding. In your code, variable padding ends up being assigned to the stride argument.
To solve the issue, you can specify the argument name with a keyword argument, i.e. padding=padding. This disambiguates the issue with positional arguments stride and padding.
class CNN2(nn.Module):
    def __init__(self):
        super(CNN2, self).__init__()
        self.conv_layer1 = self._conv_layer(3, 12)

    def _conv_layer(self, in_c, out_c, conv_kernel_size=3, padding=0):
        layer = nn.Sequential(
          nn.Conv3d(in_c, out_c, conv_kernel_size, padding=padding))
        return layer

    def forward(self, x):
        out = self.conv_layer1(x)
        return out


Instruction: Difference between autograd.grad and autograd.backward?

Input:
Suppose I have my custom loss function and I want to fit the solution of some differential equation with help of my neural network. So in each forward pass, I am calculating the output of my neural net and then calculating the loss by taking the MSE with the expected equation to which I want to fit my perceptron.
Now my doubt is: should I use grad(loss) or should I do loss.backward() for backpropagation to calculate and update my gradients?
I understand that while using loss.backward() I have to wrap my tensors with Variable and have to set the requires_grad = True for the variables w.r.t which I want to take the gradient of my loss.
So my questions are :

Does grad(loss) also requires any such explicit parameter to identify the variables for gradient computation?
How does it actually compute the gradients?
Which approach is better?
what is the main difference between the two in a practical scenario.

It would be better if you could explain the practical implications of both approaches because whenever I try to find it online I am just bombarded with a lot of stuff that isn't much relevant to my project.

Output:
In addition to Ivan's answer, having torch.autograd.grad not accumulating gradients into .grad can avoid racing conditions in multi-thread scenarios.
Quoting PyTorch doc https://pytorch.org/docs/stable/notes/autograd.html#non-determinism

If you are calling backward() on multiple thread concurrently but with shared inputs (i.e. Hogwild CPU training). Since parameters are automatically shared across threads, gradient accumulation might become non-deterministic on backward calls across threads, because two backward calls might access and try to accumulate the same .grad attribute. This is technically not safe, and it might result in racing condition and the result might be invalid to use.

But this is expected pattern if you are using the multithreading approach to drive the whole training process but using shared parameters, user who use multithreading should have the threading model in mind and should expect this to happen. User could use the functional API torch.autograd.grad() to calculate the gradients instead of backward() to avoid non-determinism.

implementation details https://github.com/pytorch/pytorch/blob/7e3a694b23b383e38f5e39ef960ba8f374d22404/torch/csrc/autograd/functions/accumulate_grad.h


Instruction: What is the alias to pytorch NN.module in TensorFlow?

Input:
I am trying to implement Triplet attention in TensorFlow. One of the question I am facing is what to use in place of NN.module in TensorFlow
class ChannelPool(nn.Module):
    def forward(self, x):
        return torch.cat( (torch.max(x,1)[0].unsqueeze(1), torch.mean(x,1).unsqueeze(1)), dim=1)

What do I put in Place of nn.Module here ?

Output:
In this case, nn.Module is used to create a custom layer. TensorFlow has a tutorial on that, please take a look. In short, one way you could implement it is with tf.keras.layers.Layer, where call is the equivalent of forward in PyTorch:
class ChannelPool(tf.keras.layers.Layer):
    def call(self, inputs):
        return tf.concat((tf.reduce_max(inputs, axis=1, keepdims=True), tf.reduce_mean(inputs, axis=1, keepdims=True)), axis=1)

You can check that they are equivalent like this:
import torch
from torch import nn
import tensorflow as tf
import numpy as np

class PyTorch_ChannelPool(nn.Module):
    def forward(self, x):
        return torch.cat((torch.max(x, 1)[0].unsqueeze(1), torch.mean(x, 1).unsqueeze(1)), dim=1)

class TensorFlow_ChannelPool(tf.keras.layers.Layer):
    def call(self, inputs):
        return tf.concat((tf.reduce_max(inputs, axis=1, keepdims=True), tf.reduce_mean(inputs, axis=1, keepdims=True)), axis=1)

np.random.seed(2021)
x = np.random.random((1,2,3,4)).astype(np.float32)

a = PyTorch_ChannelPool()
b = TensorFlow_ChannelPool()

pytorch_output = a(torch.from_numpy(x)).numpy()
tensorflow_output = b(x).numpy()

np.all(pytorch_output == tensorflow_output)
# >>> True


Instruction: Pooling for 1D tensor

Input:
I am looking for a way to reduce the length of a 1D tensor by applying a pooling operation. How can I do it? If I apply MaxPool1d, I get the error max_pool1d() input tensor must have 2 or 3 dimensions but got 1.
Here is my code:
import numpy as np
import torch

A = np.random.rand(768)
m = nn.MaxPool1d(4,4)
A_tensor = torch.from_numpy(A)
output = m(A_tensor)

Output:
Your initialization is fine, you've defined the first two parameters of nn.MaxPool1d: kernel_size and stride. For one-dimensional max-pooling both should be integers, not tuples.
The issue is with your input, it should be two-dimensional (the batch axis is missing):
>>> m = nn.MaxPool1d(4, 4)
>>> A_tensor = torch.rand(1, 768)

Then inference will result in:
>>> output = m(A_tensor)
>>> output.shape
torch.Size([1, 192])


Instruction: using ImageFolder with albumentations in pytorch

Input:
I have a situation where I need to use ImageFolder with the albumentations lib to make the augmentations in pytorch - custom dataloader is not an option.
To this end, I am stumped and I am not able to get ImageFolder to work with albumenations. I have tried something along these lines:
class Transforms:
    def __init__(self, transforms: A.Compose):
        self.transforms = transforms

    def __call__(self, img, *args, **kwargs):
        return self.transforms(image=np.array(img))['image']

and then:
    trainset = datasets.ImageFolder(traindir,transform=Transforms(transforms=A.Resize(32 , 32)))

where traindir is some dir with images. I however get thrown a weird error:
RuntimeError: Given groups=1, weight of size [16, 3, 3, 3], expected input[1024, 32, 32, 3] to have 3 channels, but got 32 channels instead

and I cant seem to find a reproducible example to make a simple aug pipleline work with imagefolder.
UPDATE
On the recommendation of @Shai, I have done this now:
class Transforms:
    def __init__(self):
        self.transforms = A.Compose([A.Resize(224,224),ToTensorV2()])

    def __call__(self, img, *args, **kwargs):
        return self.transforms(image=np.array(img))['image']

trainset = datasets.ImageFolder(traindir,transform=Transforms())

but I get thrown:
    self.padding, self.dilation, self.groups)
RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

Output:
You need to use ToTensorV2 transformation as the final one:
trainset = datasets.ImageFolder(traindir,transform=Transforms(transforms=A.Compose([A.Resize(32 , 32), ToTensorV2()]))


Instruction: How to get confidence score from a trained pytorch model

Input:
I have a trained PyTorch model and I want to get the confidence score of predictions in range (0-100) or (0-1). The code below is giving me a score but its range is undefined. I want the score in a defined range of (0-1) or (0-100). Any idea how to get this?
conf, classes = torch.max(output, 1)

My code:
model = torch.load(r'best.pt')
model.eval()
def preprocess(imgs):
    im = torch.from_numpy(imgs)
    im = im.float()  # uint8 to fp16/32
    im /= 255.0
    return im

img_path = cv2.imread("/content/634282.jpg",0)
cropped = cv2.resize(img_path,(28,28))
imgs = preprocess(np.array([[cropped]]))
def predict_allCharacters(imgs):
    output = model(imgs)
    conf, classes = torch.max(output, 1)
    class_names = '0123456789'
    return conf, class_names[classes.item()]

Model definition:
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(
                in_channels=1,
                out_channels=16,
                kernel_size=5,
                stride=1,
                padding=2,
            ),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(16, 32, 5, 1, 2),
            nn.ReLU(),
            nn.MaxPool2d(2),
        )
        # fully connected layer, output 10 classes
        self.out = nn.Linear(32 * 7 * 7, 37)
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        # flatten the output of conv2 to (batch_size, 32 * 7 * 7)
        x = x.view(x.size(0), -1)
        output = self.out(x)
        return output    # return x for visualization

Output:
In your case, output represents the logits. One way of getting a probability out of them is to use the Softmax function. As it seems that output contains the outputs from a batch, not a single sample, you can do something like this:
probs = torch.nn.functional.softmax(output, dim=1)

Then, in probs, each row would have the probability (i.e., in range [0, 1], sum=1) of each class for a given sample.
So, your predict_allCharacters could be modified to:
def predict_allCharacters(imgs):
    output = model(imgs)
    probs = torch.nn.functional.softmax(output, dim=1)
    conf, classes = torch.max(probs, 1)
    class_names = '0123456789'
    return conf, class_names[classes.item()]


Instruction: Very high CPU usage when using opencv2 with multithreading in python

Input:
I am trying to create automatic attendance system with opencv2 in which i need to get rtsp stream from IP camera, find faces from it and recognize face.
I created different threads from frame catching and drawing because face recognition function needs some time to recognize face.
But just creating 2 threads, one for frame reading and other for drawing uses around 70% CPU.
and creating pytorch_facenet model increase usage 80-90% CPU.
does anyone know how to reduce CPU usage ?
my program:
import cv2
import threading
from facenet_pytorch import InceptionResnetV1

cap = cv2.VideoCapture("rtsp://test:Test12345@125.0.0.1")
resnet = InceptionResnetV1(pretrained='vggface2').eval()

ret, frame = cap.read()
exit = False

def th1():
    global ret, frame, exit
    while True:
        ret, frame = cap.read()
        if exit:
            break

def th2():
    global ret, frame, exit
    while True:
        cv2.imshow('frame', frame)
        cv2.waitKey(1)
        if cv2.getWindowProperty('frame',cv2.WND_PROP_VISIBLE) < 1:
            exit = True
            break

t1 = threading.Thread(target=th1)
t1.start()
t2 = threading.Thread(target=th2)
t2.start()

Update:
I used time.sleep(0.2) in my all threads except frame reading.
and it worked, my cpu usage is 30% now.

Output:
Two issues.

th2 runs in an almost-tight-loop. It won't consume a whole core of CPU because waitKey(1) sleeps for some time.

No synchronization at all between threads, but you need it. You need a threading.Event to notify the consumer thread of a fresh frame. The consumer thread must wait until a fresh frame is available, because it's pointless to display the same old frame again and again. You can be lazy and use waitKey(30) instead. For the displaying thread, that's good enough.

VideoCapture. You don't do any error checking at all! You must check:

cap = cv2.VideoCapture("rtsp://test:Test12345@125.0.0.1")
assert cap.isOpened()
...

and
    while True:
        ret, frame = cap.read()
        if not ret:
            break
        ...


Instruction: Follow-up to "In PyTorch how are layer weights and biases initialized by default?"

Input:
In the answer with most voted of this question, it says:

Most layers are initialized using Kaiming Uniform method. Example layers include Linear, Conv2d, RNN etc.

I was actually wondering: Where does one know this from? For example, I would like to know the default initialization of torch.nn.Conv2d and torch.nn.BatchNorm2d for PyTorch 1.9.0. For torch.nn.Linear, I found the answer here (from the second answer of the above mentioned question).

Output:
Convolutional modules such as nn.Conv1d, nn.Conv2d, and nn.Conv3d inherit from the _ConvNd class. This class has a reset_parameters function implemented just like nn.Linear:
def reset_parameters(self) -> None:
    # Setting a=sqrt(5) in kaiming_uniform is the same as initializing with
    # uniform(-1/sqrt(k), 1/sqrt(k)), where k = weight.size(1) * prod(*kernel_size)
    # For more details see:
    # https://github.com/pytorch/pytorch/issues/15314#issuecomment-477448573
    init.kaiming_uniform_(self.weight, a=math.sqrt(5))
    if self.bias is not None:
        fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
        bound = 1 / math.sqrt(fan_in)
        init.uniform_(self.bias, -bound, bound)

As for nn.BatchNorm2d, it has reset_parameters and reset_running_stats function:
def reset_parameters(self) -> None:
    self.reset_running_stats()
    if self.affine:
        init.ones_(self.weight)
        init.zeros_(self.bias)

def reset_running_stats(self) -> None:
    if self.track_running_stats:
        # running_mean/running_var/num_batches... are registered at runtime depending
        # if self.track_running_stats is on
        self.running_mean.zero_()  # type: ignore[operator]
        self.running_var.fill_(1)  # type: ignore[operator]
        self.num_batches_tracked.zero_()  # type: ignore[operator]


Instruction: One of the variables modified by an inplace operation

Input:
I am relatively new to Pytorch. Here I want to use this model to generate some images, however as this was written before Pytorch 1.5, since the gradient calculation has been fixed then, this is the error message.
RuntimeError: one of the variables needed for gradient computation has been
modified by an inplace operation: [torch.cuda.FloatTensor [1, 512, 4, 4]]
is at version 2; expected version 1 instead.
Hint: enable anomaly detection to find the operation that
failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

I have looked at past examples and am not sure what is the problem here, I believe it is happening within this region but I don’t know where! Any help would be greatly appreciated!
def process(self, images, edges, masks):
self.iteration += 1

    # zero optimizers
    self.gen_optimizer.zero_grad()
    self.dis_optimizer.zero_grad()

    # process outputs
    outputs = self(images, edges, masks)
    gen_loss = 0
    dis_loss = 0

    # discriminator loss
    dis_input_real = torch.cat((images, edges), dim=1)
    dis_input_fake = torch.cat((images, outputs.detach()), dim=1)
    dis_real, dis_real_feat = self.discriminator(dis_input_real)        # in: (grayscale(1) + edge(1))
    dis_fake, dis_fake_feat = self.discriminator(dis_input_fake)        # in: (grayscale(1) + edge(1))
    dis_real_loss = self.adversarial_loss(dis_real, True, True)
    dis_fake_loss = self.adversarial_loss(dis_fake, False, True)
    dis_loss += (dis_real_loss + dis_fake_loss) / 2

    # generator adversarial loss
    gen_input_fake = torch.cat((images, outputs), dim=1)
    gen_fake, gen_fake_feat = self.discriminator(gen_input_fake)        # in: (grayscale(1) + edge(1))
    gen_gan_loss = self.adversarial_loss(gen_fake, True, False)
    gen_loss += gen_gan_loss

    # generator feature matching loss
    gen_fm_loss = 0
    for i in range(len(dis_real_feat)):
        gen_fm_loss += self.l1_loss(gen_fake_feat[i], dis_real_feat[i].detach())
    gen_fm_loss = gen_fm_loss * self.config.FM_LOSS_WEIGHT
    gen_loss += gen_fm_loss

    # create logs
    logs = [
        ("l_d1", dis_loss.item()),
        ("l_g1", gen_gan_loss.item()),
        ("l_fm", gen_fm_loss.item()),
    ]

    return outputs, gen_loss, dis_loss, logs

def forward(self, images, edges, masks):
    edges_masked = (edges * (1 - masks))
    images_masked = (images * (1 - masks)) + masks
    inputs = torch.cat((images_masked, edges_masked, masks), dim=1)
    outputs = self.generator(inputs)                                    # in: [grayscale(1) + edge(1) + mask(1)]
    return outputs

def backward(self, gen_loss=None, dis_loss=None):
    if dis_loss is not None:
        dis_loss.backward()
    self.dis_optimizer.step()

    if gen_loss is not None:
        gen_loss.backward()
    self.gen_optimizer.step()

Thank you!

Output:
You can't compute the loss for the discriminator and for the generator in one go and have the both back-propagations back-to-back like this:
if dis_loss is not None:
    dis_loss.backward()
self.dis_optimizer.step()

if gen_loss is not None:
    gen_loss.backward()
self.gen_optimizer.step()

Here's the reason why: when you call self.dis_optimizer.step(), you effectively in-place modify the parameters of the discriminator, the very same that were used to compute gen_loss which you are trying to backpropagate on. This is not possible.
You have to compute dis_loss backpropagate, update the weights of the discriminator, and clear the gradients. Only then can you compute gen_loss with the newly updated discriminator weights. Finally, backpropagate on the generator.
This tutorial is a good walkthrough over a typical GAN training.


Instruction: Function AddmmBackward returned an invalid gradient

Input:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import matplotlib.pyplot as plt
import numpy as np
import torch.optim as optim

class NeuralNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 3)
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = NeuralNetwork()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

from torchvision import datasets, transforms
from torch.utils.data import DataLoader, random_split

def UploadData(path, train):
    #set up transforms for train and test datasets
    train_transforms = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize(255), transforms.CenterCrop(224), transforms.RandomRotation(30),
                                         transforms.RandomHorizontalFlip(), transforms.transforms.ToTensor()])
    valid_transforms = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize(255), transforms.CenterCrop(224), transforms.RandomRotation(30),
                                         transforms.RandomHorizontalFlip(), transforms.transforms.ToTensor()])
    test_transforms = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize(255), transforms.CenterCrop(224), transforms.ToTensor()])

    #set up datasets from Image Folders
    train_dataset = datasets.ImageFolder(path + '/train', transform=train_transforms)
    valid_dataset = datasets.ImageFolder(path + '/validation', transform=valid_transforms)
    test_dataset = datasets.ImageFolder(path + '/test', transform=test_transforms)

    #set up dataloaders with batch size of 32
    trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)
    validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=32, shuffle=True)
    testloader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=True)

    return trainloader, validloader, testloader

trainloader, validloader, testloader = UploadData("/home/lns/research/dataset", True)

epochs = 5
min_valid_loss = np.inf
for e in range(epochs):
    train_loss = 0.0
    for data, labels in trainloader:
        # Transfer Data to GPU if available
        if torch.cuda.is_available():
            print("using GPU for data")
            data, labels = data.cuda(), labels.cuda()

        # Clear the gradients
        optimizer.zero_grad()
        # Forward Pass
        target = net(data)
        # Find the Loss
        loss = criterion(target,labels)
        # Calculate gradients
        loss.backward()
        # Update Weights
        optimizer.step()
        # Calculate Loss
        train_loss += loss.item()

    valid_loss = 0.0
    model.eval()     # Optional when not using Model Specific layer
    for data, labels in validloader:
        # Transfer Data to GPU if available
        if torch.cuda.is_available():
            print("using GPU for data")
            data, labels = data.cuda(), labels.cuda()

        # Forward Pass
        target = net(data)
        # Find the Loss
        loss = criterion(target,labels)
        # Calculate Loss
        valid_loss += loss.item()

    print('Epoch ',e+1, '\t\t Training Loss: ',train_loss / len(trainloader),' \t\t Validation Loss: ',valid_loss / len(validloader))

    if min_valid_loss > valid_loss:
        print("Validation Loss Decreased(",min_valid_loss,"--->",valid_loss,") \t Saving The Model")
        min_valid_loss = valid_loss

        # Saving State Dict
        torch.save(net.state_dict(), '/home/lns/research/MODEL.pth')

After searching a lot i am asking for help. Can someone help me
understand why this error is occuring in backward propagation.
i followed pytorch cnn tutorail and geeksforgeeks tutorial
dataset is x ray images transformed into grayscale and resize to 255
Is my neural network is wrong or data is not processed correctly?

Output:
This is a size mismmatch between the output of your CNN and the number of neurons on on your first fully-connected layer. Because of missing padding, the number of elements when flattened is 16*4*4 i.e. 256 (and not 16*5*5):
self.fc1 = nn.Linear(256, 120)

Once modified, the model will run correctly:
>>> model = NeuralNetwork()
>>> model(torch.rand(1, 1, 28, 28)).shape
torch.Size([1, 3])

Alternatively, you can use an nn.LazyLinear which will deduce the in_feature argument during the very first inference based on its input shape.
self.fc1 = nn.LazyLinear(120)


Instruction: Getting error with Pytorch lightning when passing model checkpoint

Input:
I am training a multi-label classification problem using Hugging face models. I am using Pytorch Lightning to train the model.
Here is the code:
And early stopping triggers when the loss hasn't improved for the last
early_stopping_callback = EarlyStopping(monitor='val_loss', patience=2)

We can start the training process:
checkpoint_callback = ModelCheckpoint(
  dirpath="checkpoints",
  filename="best-checkpoint",
  save_top_k=1,
  verbose=True,
  monitor="val_loss",
  mode="min"
)

trainer = pl.Trainer(
  logger=logger,
  callbacks=[early_stopping_callback],
  max_epochs=N_EPOCHS,
 checkpoint_callback=checkpoint_callback,
  gpus=1,
  progress_bar_refresh_rate=30
)
# checkpoint_callback=checkpoint_callback,

As soon as I run this, I get this error:
~/.local/lib/python3.6/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py in _configure_checkpoint_callbacks(self, checkpoint_callback)
     75             if isinstance(checkpoint_callback, Callback):
     76                 error_msg += " Pass callback instances to the `callbacks` argument in the Trainer constructor instead."
---> 77             raise MisconfigurationException(error_msg)
     78         if self._trainer_has_checkpoint_callbacks() and checkpoint_callback is False:
     79             raise MisconfigurationException(

MisconfigurationException: Invalid type provided for checkpoint_callback: Expected bool but received <class 'pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint'>. Pass callback instances to the `callbacks` argument in the Trainer constructor instead.

How can I fix this issue?

Output:
You can look up the description of the checkpoint_callback argument in the documentation page of pl.Trainer:

checkpoint_callback (bool) – If True, enable checkpointing. It will configure a default ModelCheckpoint callback if there is no user-defined ModelCheckpoint in callbacks.

You shouldn't pass your custom ModelCheckpoint to this argument. I believe what you are looking to do is to pass both the EarlyStopping and ModelCheckpoint in the callbacks list:
early_stopping_callback = EarlyStopping(monitor='val_loss', patience=2)

checkpoint_callback = ModelCheckpoint(
    dirpath="checkpoints",
    filename="best-checkpoint",
    save_top_k=1,
    verbose=True,
    monitor="val_loss",
    mode="min")

trainer = pl.Trainer(
    logger=logger,
    callbacks=[checkpoint_callback, early_stopping_callback],
    max_epochs=N_EPOCHS,
    gpus=1,
    progress_bar_refresh_rate=30)


Instruction: potential bug when upgrading to PyTorch 1.9 ImportError: cannot import name 'int_classes' from 'torch._six'

Input:
I installed PyTorch using
$ pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
I get this error:
(proxy) [jalal@goku proxynca_pp]$ CUDA_VISIBLE_DEVICES=0,1 python train.py --dataset cub  --config config/cub.json --mode train --apex --seed 0
Traceback (most recent call last):
  File "train.py", line 3, in <module>
    import dataset
  File "/scratch3/research/code/fashion/proxynca_pp/dataset/__init__.py", line 6, in <module>
    from . import utils
  File "/scratch3/research/code/fashion/proxynca_pp/dataset/utils.py", line 8, in <module>
    from torch._six import int_classes as _int_classes
ImportError: cannot import name 'int_classes' from 'torch._six' (/scratch3/venv/proxy/lib/python3.8/site-packages/torch/_six.py)

The code is from this GitHub repo.
What in the code should I change to get it working?
(proxy) [jalal@goku proxynca_pp]$ pip freeze
h5py==3.4.0
numpy==1.21.2
Pillow==8.3.2
scipy==1.7.1
torch==1.9.0+cu111
torchaudio==0.9.0
torchvision==0.10.0+cu111
tqdm==4.62.2
typing-extensions==3.10.0.2

Output: Easiest solution would be to just set int_classes = int instead of importing it from _six.


Instruction: installing NVIDIA Apex for Python 3.8.5 and compatible with PyTorch 1.9

Input:
I am running a code that apparently requires NVIDIA apex (I initially didn't know and installed the wrong apex). I am unsure how to fix the final error:
(proxy) [jalal@goku proxynca_pp]$ CUDA_VISIBLE_DEVICES=0,1 python train.py --dataset cub  --config config/cub.json --mode train --apex --seed 0
(1024, 4096)
train.py:12: MatplotlibDeprecationWarning: The 'warn' parameter of use() is deprecated since Matplotlib 3.1 and will be removed in 3.3.  If any parameter follows 'warn', they should be pass as keyword, not positionally.
  matplotlib.use('agg', warn=False, force=True)
Traceback (most recent call last):
  File "train.py", line 70, in <module>
    from apex import amp
  File "/scratch3/venv/proxy/lib/python3.8/site-packages/apex/__init__.py", line 13, in <module>
    from pyramid.session import UnencryptedCookieSessionFactoryConfig
ImportError: cannot import name 'UnencryptedCookieSessionFactoryConfig' from 'pyramid.session' (unknown location)

After I got the above error, I tried this answer: https://stackoverflow.com/a/67188946/2414957
(proxy) [jalal@goku proxynca_pp]$ pip uninstall apex
Found existing installation: apex 0.9.10.dev0
Uninstalling apex-0.9.10.dev0:
  Would remove:
    /scratch3/venv/proxy/lib/python3.8/site-packages/apex-0.9.10.dev0-py3.8.egg-info
    /scratch3/venv/proxy/lib/python3.8/site-packages/apex/*
Proceed (Y/n)? y
  Successfully uninstalled apex-0.9.10.dev0
(proxy) [jalal@goku proxynca_pp]$ git clone https://github.com/NVIDIA/apex
Cloning into 'apex'...
remote: Enumerating objects: 8256, done.
remote: Counting objects: 100% (343/343), done.
remote: Compressing objects: 100% (192/192), done.
remote: Total 8256 (delta 204), reused 240 (delta 139), pack-reused 7913
Receiving objects: 100% (8256/8256), 14.20 MiB | 0 bytes/s, done.
Resolving deltas: 100% (5605/5605), done.
(proxy) [jalal@goku proxynca_pp]$ cd apex
(proxy) [jalal@goku apex]$ pip install -v --disable-pip-version-check --no-cache-dir \
> --global-option="--cpp_ext" --global-option="--cuda_ext" ./
/scratch3/venv/proxy/lib/python3.8/site-packages/pip/_internal/commands/install.py:229: UserWarning: Disabling all use of wheels due to the use of --build-option / --global-option / --install-option.
  cmdoptions.check_install_build_global(options)
Using pip 21.2.4 from /scratch3/venv/proxy/lib/python3.8/site-packages/pip (python 3.8)
Processing /scratch3/research/code/fashion/proxynca_pp/apex
  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.
   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.
    Running command python setup.py egg_info

    torch.__version__  = 1.9.0+cu111

    running egg_info
    creating /scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info
    writing /scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/PKG-INFO
    writing dependency_links to /scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/dependency_links.txt
    writing top-level names to /scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/top_level.txt
    writing manifest file '/scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/SOURCES.txt'
    reading manifest file '/scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/SOURCES.txt'
    writing manifest file '/scratch/tmp/pip-pip-egg-info-yc32vm37/apex.egg-info/SOURCES.txt'
    /scratch/tmp/pip-req-build-fg_khhkt/setup.py:67: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!
      warnings.warn("Option --pyprof not specified. Not installing PyProf dependencies!")
Skipping wheel build for apex, due to binaries being disabled for it.
Installing collected packages: apex
    Running command /scratch3/venv/proxy/bin/python3.8 -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '"'"'/scratch/tmp/pip-req-build-fg_khhkt/setup.py'"'"'; __file__='"'"'/scratch/tmp/pip-req-build-fg_khhkt/setup.py'"'"';f = getattr(tokenize, '"'"'open'"'"', open)(__file__) if os.path.exists(__file__) else io.StringIO('"'"'from setuptools import setup; setup()'"'"');code = f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' --cpp_ext --cuda_ext install --record /scratch/tmp/pip-record-u812zb2v/install-record.txt --single-version-externally-managed --compile --install-headers /scratch3/venv/proxy/include/site/python3.8/apex

    torch.__version__  = 1.9.0+cu111

    /scratch/tmp/pip-req-build-fg_khhkt/setup.py:67: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!
      warnings.warn("Option --pyprof not specified. Not installing PyProf dependencies!")

    Compiling cuda extensions with
    nvcc: NVIDIA (R) Cuda compiler driver
    Copyright (c) 2005-2018 NVIDIA Corporation
    Built on Sat_Aug_25_21:08:01_CDT_2018
    Cuda compilation tools, release 10.0, V10.0.130
    from /usr/local/cuda-10.0/bin

    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/scratch/tmp/pip-req-build-fg_khhkt/setup.py", line 159, in <module>
        check_cuda_torch_binary_vs_bare_metal(CUDA_HOME)
      File "/scratch/tmp/pip-req-build-fg_khhkt/setup.py", line 99, in check_cuda_torch_binary_vs_bare_metal
        raise RuntimeError("Cuda extensions are being compiled with a version of Cuda that does " +
    RuntimeError: Cuda extensions are being compiled with a version of Cuda that does not match the version used to compile Pytorch binaries.  Pytorch binaries were compiled with Cuda 11.1.
    In some cases, a minor-version mismatch will not cause later errors:  https://github.com/NVIDIA/apex/pull/323#discussion_r287021798.  You can try commenting out this check (at your own risk).
    Running setup.py install for apex ... error
ERROR: Command errored out with exit status 1: /scratch3/venv/proxy/bin/python3.8 -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '"'"'/scratch/tmp/pip-req-build-fg_khhkt/setup.py'"'"'; __file__='"'"'/scratch/tmp/pip-req-build-fg_khhkt/setup.py'"'"';f = getattr(tokenize, '"'"'open'"'"', open)(__file__) if os.path.exists(__file__) else io.StringIO('"'"'from setuptools import setup; setup()'"'"');code = f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' --cpp_ext --cuda_ext install --record /scratch/tmp/pip-record-u812zb2v/install-record.txt --single-version-externally-managed --compile --install-headers /scratch3/venv/proxy/include/site/python3.8/apex Check the logs for full command output.

I have these packages installed:
(proxy) [jalal@goku apex]$ pip freeze
anykeystore==0.2
certifi==2021.5.30
charset-normalizer==2.0.4
cryptacular==1.6.2
cycler==0.10.0
defusedxml==0.7.1
greenlet==1.1.1
h5py==3.4.0
hupper==1.10.3
idna==3.2
joblib==1.0.1
kiwisolver==1.3.2
MarkupSafe==2.0.1
matplotlib==3.2.0
numpy==1.21.2
oauthlib==3.1.1
PasteDeploy==2.1.1
pbkdf2==1.3
Pillow==8.3.2
plaster==1.0
plaster-pastedeploy==0.7
pyparsing==2.4.7
pyramid==2.0
pyramid-mailer==0.15.1
python-dateutil==2.8.2
python3-openid==3.2.0
repoze.sendmail==4.4.1
requests==2.26.0
requests-oauthlib==1.3.0
scikit-learn==0.24.2
scipy==1.7.1
six==1.16.0
sklearn==0.0
SQLAlchemy==1.4.23
threadpoolctl==2.2.0
torch==1.9.0+cu111
torchaudio==0.9.0
torchvision==0.10.0+cu111
tqdm==4.62.2
transaction==3.0.1
translationstring==1.4
typing-extensions==3.10.0.2
urllib3==1.26.6
velruse==1.1.1
venusian==3.0.0
WebOb==1.8.7
WTForms==2.3.3
wtforms-recaptcha==0.3.2
zope.deprecation==4.4.0
zope.interface==5.4.0
zope.sqlalchemy==1.6

And here's the code is from this GitHub repo.
Edit: I found the steps through a stackoverflow answer that I can't find now (linked above). I don't know how to find the proper link or installation that is compatible with PyTorch 1.9.
FYI, the git repo has no installation instruction hence I am installing things blindly.

Output:
Installing CUDA 11.1 and then adding the following to ~/.bashrc and sourcing the ~/.bashrc and finally the symlink made it work:
export CUDA_HOME=/usr/local/cuda-11.1
export PATH=/usr/local/cuda-11.1/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH

This eliminates the need to uninstall CUDA 10.2 especially if needed later for other project. Simply exporting the path and not using the symlink didn't work.
$ sudo ln -sfT /usr/local/cuda/cuda-11.1/ /usr/local/cuda
^ Last command is assuming you have multiple CUDA versions installed in your machine.
For further information please read this GitHub issue.


Instruction: How to calculate (N,*, input) matrix class (output, input) matrix(the result is (N,*,output)) by pytorch?

Input:
I want to rewrite what nn.Linear do. The question is that the input size is (N, *,in_feature) and weight size is (out_feature, in_feature). If I want the result to be (N,*,out_feature) using python, how should I wirte the code?
input @ weight.T

is not right, sadly.

Output:
The sizes need to match in order to apply @, i.e. __matmul__: the input x is shaped (N, *, in_feature) and the weight tensor w is shaped (out_feature, in_feature).
x = torch.rand(2, 4, 4, 10)
w = torch.rand(5, 10)

Taking the transpose of w will get you a shape of (in_feature, out_feature). Applying __matmul__ between x and w.T will reduce down to a shape of (N, *, out_feature):
>>> z = x@w.T
>>> z.shape
torch.Size([2, 4, 4, 5])

Or equivalently using torch.matmul:
>>> z = torch.matmul(x, w.T)


Instruction: PyTorch: How to normalize a tensor when the image is cropped randomly?

Input:
Let's say we are working with the CIFAR-10 dataset and we want to apply some data augmentation and additionally normalize the tensors. Here is some reproducible code for this
from torchvision import transforms, datasets
import matplotlib.pyplot as plt
trafo = transforms.Compose([transforms.Pad(padding = 4, fill = 0, padding_mode = "constant"),
                            transforms.RandomHorizontalFlip(p=0.5),
                            transforms.RandomCrop(size = (32, 32)),
                            transforms.ToTensor(),
                            transforms.Normalize(mean = (0.0, 0.0, 0.0), std = (1.0, 1.0, 1.0))]
                          )

cifar10_full = datasets.CIFAR10(root = "CIFAR-10", train = True, transform = trafo, target_transform = None, download = True)

The normalization I chose so far would do nothing with the tensors since I put the mean and std to 0 and 1 respectively. According to the documentation of torchvision.transforms.Normalize, the provided means and standard deviations are for each channel of the input. However, the problem is that that I cannot calculate the mean across each channel because of some random flipping and cropping mean. Therefore, my idea was something along the following lines
trafo_1 = transforms.Compose([transforms.Pad(padding = 4, fill = 0, padding_mode = "constant"),
                            transforms.RandomHorizontalFlip(p=0.5),
                            transforms.RandomCrop(size = (32, 32)),
                            transforms.ToTensor()
                          )

cifar10_full = datasets.CIFAR10(root = "CIFAR-10", train = True, transform = trafo_1, target_transform = None, download = True)

Now I could calculate the mean across each channel of the input and then I wanted to normalize the tensors again. However, I cannot simply use transforms.Normalize() as cifar10_full is not the original dataset anymore, but how I could proceed instead? (One solution would be to simply fix the seed of the random generators, i.e use torch.manual_seed(0), but I would like to avoid this for now...)

Output:
The mean and std are not for each tensor, but from the whole dataset. What you are trying to do doesn't really matter, you just want a scale that is good enough for the whole data representation, there is no exact mean or std you will get, these are all random operations, just use the mean and std from the actual data, which is pretty much the standard.
First, try to calculate the mean and std of the dataset (try random sampling), and use that for normalization.
# Calculate the mean, std of the complete dataset
import glob
import cv2
import numpy as np
import tqdm
import random

# calculating 3 channel mean and std for image dataset

means = np.array([0, 0, 0], dtype=np.float32)
stds = np.array([0, 0, 0], dtype=np.float32)
total_images = 0
randomly_sample = 5000
for f in tqdm.tqdm(random.sample(glob.glob("dataset_path/**.jpg", recursive = True), randomly_sample)):
    img = cv2.imread(f)
    means += img.mean(axis=(0,1))
    stds += img.std(axis=(0,1))
    total_images += 1
means = means / (total_images * 255.)
stds = stds / (total_images * 255.)
print("Total images: ", total_images)
print("Means: ", means)
print("Stds: ", stds)

Just a simple scenario, do you think in actual testing or inference your images will be augmented this way too, probably not, you will have clean images which match closely with the mean and std from the clean version of the data, so it's useless to calculate mean and std (you can take few random samples), unless you want to apply TTA.
If you want to apply TTA too, then you can go ahead and run some augmentation on the images, do random sampling and take the mean and std of those images.


Instruction: How does Learning Rate affect Gradients in Back Prop

Input:
I was wondering if the learning rate is set to update the weights during the backpropagation on the layers of a neural network. How and where do the weights get updated?
I can't see the connection between optimizer, learning rate, and the backpropagation function.

Output:
Sure a fair question that comes to mind is how are the backpropagation and the optimizer related in all of this. And how are those parameters updated by the optimizer?
optimizer.zero_grad()
loss.backward()
optimizer.step()

Indeed there doesn't seem to be any link between optimizer and loss let alone the parameters of your model.

Optimizer

Every tensor object has a grad attribute which either contains a tensor representing the gradient corresponding to it or None if either it doesn't require gradient computation or simply doesn't have any gradient.
To optimize parameters in PyTorch you would go about initializing an optimizer by passing a list or iterator over those very parameters you want this optimizer to act upon:
mlp = nn.Sequential(nn.Linear(10, 2), nn.Linear(2, 1))
optimizer = torch.optim.SGD(mlp.parameters(), lr=1.0e-3)

An optimizer is not tasked with computing gradients, this is performed by another system in PyTorch.
When you call optimizer.step(), the optimizer will go over each provided parameter and update them based the optimizer's update rule (i.e. the optimizing method used) and the gradient associated with each parameter (namely the grad attribute)
For SGD it will be something like (leaving no_grad considerations aside):
for param in parameters:
    param -= lr*param.grad

Backpropagation

To actually compute the backward propagation you would use torch.autograd.backward which usually comes in the form of torch.Tensor.backward (as a torch.Tensor method).
This function is a mutable operator which will update the grad attribute of all leaf tensor nodes requiring gradient computation. In other words, it will compute the gradient of the tensor backward that was called upon w.r.t each parameter of the model.
For example with model mlp, we backpropagate on a dummy loss:
>>> for x in mlp.parameters():
...     print(tuple(x.shape), x.grad)
(2, 10) None
(2,)    None
(1, 2)  None
(1,)    None

After inference and backpropagation on a random input:
>>> mlp(torch.rand(1, 10)).mean().backward()

Each grad attribute of the model's tensor parameter has been updated by that call:
>>> for x in mlp.parameters():
...    print(tuple(x.shape), x.grad is not None, tuple(x.grad.shape))
(2, 10) True (2, 10)
(2,)    True (2,)
(1, 2)  True (1, 2)
(1,)    True (1,)

Then you can call optimizer.step() to effectively perform the parameter update based on those gradients. Do note, the optimizer can only affect tensors that have been provided to it on initialization (recall the torch.optim.SGD(mlp.parameters(), lr=1.0e-3) part).
Finally you can zero the gradient of those parameter from the optimizer directly with zero_grad:
>>> optimizer.zero_grad()

This is roughly a shorthand for:
for param in mtl.parameters():
    param.grad.zero_()

But its effectiveness is a lot more apparent when using multiple parameters groups and or multiple optimizers on the same model.


Instruction: Why do repeated calls to torch.cuda.is_available() all return True?

Input:
I'm reading code that makes multiple calls to torch.cuda.is_available(). Each time it prepares to query a net, calls a function (shown below) that uses torch.cuda.is_available() to set the device (cuda or cpu) used. I don't understand why calls after the first one don't return False, thus pushing computation to the cpu.
Is the GPU released when the code leaves the method? Or, does each call take up only a relatively small part of the GPU, so that the code would need to make multiple calls to this method, before computation was pushed to the CPU?
Code in question:
def computeProposals(imageName):
    app.config['args'].img = imageName
    print('ARGs --img = ', app.config['args'].img)

    # Setup device
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    # Setup Model
    Config = namedtuple('Config', ['iSz', 'oSz', 'gSz', 'batch'])
    config = Config(iSz=160, oSz=56, gSz=112, batch=1)  # default for training

    model = (models.__dict__[app.config['args'].arch](config))
    model = load_pretrain(model, app.config['args'].resume)
    model = model.eval().to(device)

    scales_range = np.arange(app.config['args'].si,
                             app.config['args'].sf + app.config['args'].ss,
                             app.config['args'].ss)
    scales = [2 ** i for i in scales_range]
    meanstd = {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}
    infer = Infer(nps=app.config['args'].nps, scales=scales, meanstd=meanstd,
                  model=model, device=device)

    print('| start')
    tic = time.time()
    im = np.array(Image.open(app.config['args'].img).convert('RGB'),
              dtype=np.float32)
    h, w = im.shape[:2]
    img = np.expand_dims(np.transpose(im, (2, 0, 1)), axis=0).astype(np.float32)
    img = torch.from_numpy(img / 255.).to(device)
    infer.forward(img)
    masks, scores = infer.getTopProps(.2, h, w)
    toc = time.time() - tic
    print('| done in %05.3f s' % toc)

   return masks, scores

Here is the code that makes the repeated calls to computeProposals:
@app.route('/')
def index():
    global cc_data

    base_dir = app.config['base_dir']
    img_name = app.config['img_name']

    img_dir = os.path.join(base_dir, 'images')
    img_path = os.path.join(img_dir, img_name)
    print(img_path)
    print('Loading image and proposals, please wait')

    img = skio.imread(img_path)
    img = img[:, :, :3]

    masks, scores = computeProposals(img_path)

    session['pos_wts'] = np.zeros(masks.shape[2], dtype=np.float64).tolist()
    session['neg_wts'] = np.zeros(masks.shape[2], dtype=np.float64).tolist()
    masks = np.transpose(masks, (2, 0, 1))
    dilated = dilate_proposals(masks)
    print('Loading done')

    img_h = img.shape[0]
    img_w = img.shape[1]
    print('Image height {} and width {}'.format(img_h, img_w))

    rendered_img = draw_buttons(np.copy(img), img_w)
    if app.config['DoneFlag'] == 1:
        rendered_img = draw_end(rendered_img, img_h)

    img_stream = embed_image_html(rendered_img)

    # Create dicts with session variables
    cc_data = {'img_h': img_h, 'img_w': img_w, 'masks': masks,
               'scores': scores, 'dilated': dilated,
               'orig': np.copy(img).tolist(), 'render': rendered_img.tolist(),
               'clicks': []}

    session['response'] = {'input_img': img_stream,
                           'im_width': img_w, 'im_height': img_h,
                           'show_error': False}

    return render_template('index.html', response=session['response'])

Output: Function torch.cuda.is_available() does not return whether the Cuda device is being used or if there is a memory left on the (those) device(s). This means the returned value will not depend on the number of processes running or the memory which has already been allocated to those processes. It only returns whether one or more Cuda devices are accessible by PyTorch.


Instruction: Background removal with U2Net is too strong

Input:
I am successfully using U2Net to remove the background of images using the terminal and I am also using the nice interface of this repo to do the same thing just in an easier way and validate the similarity of the results. However, my issue is that the background removal is too strong for images like this:

Where I get the following result (i.e. packaging is also removed):

If I upload the image in Foco clipping website and I select Type=='Graphic' I get exactly the same results. That means that the website is using the same algorithm to remove the background for Graphic-type images. Nevertheless, if I select Type=='Product', then the result is the following and is exactly what I want:

Does anyone have any idea on what to do to obtain the same result?

Output:
you should sharpen the image first  (use strong sharpen)

from PIL import Image
from PIL import ImageFilter
from PIL.ImageFilter import (UnsharpMask)
simg = Image.open('data/srcimg07.jpg')
dimg = simg.filter(UnsharpMask(radius=4.5, percent=200, threshold=0))
dimg.save("result/ImageFilter_UnsharpMask_2_150_3.jpg")

use U2Net to remove the background of the sharpened image

use the result from step (2) as mask

using the mask from step (3) extract wanted result from original picture

note: this is very quick example, you could refine it more


Instruction: How can I concatenate an additional input data in Alexnet with the output of the last dropout layer using Pytorch implementation?

Input:
Here is the implementation architecture
class AlexNet(nn.Module):
def __init__(self, num_classes=10):
    super(AlexNet, self).__init__()
    #1
    self.features= nn.Sequential(
    nn.Conv2d(3, 96, kernel_size=11, stride=4, padding=0),
    nn.ReLU(inplace=True),
    nn.MaxPool2d(kernel_size=3, stride=2),
    #2
    nn.Conv2d(96, 256, kernel_size=5, stride=1, padding=2),
    nn.ReLU(inplace=True),
    nn.MaxPool2d(kernel_size=3, stride=2),
    #3
    nn.Conv2d(256, 384, kernel_size=3, stride=1, padding=1),
    nn.ReLU(inplace=True),
    #4
    nn.Conv2d(384, 384, kernel_size=3, stride=1, padding=1),
    nn.ReLU(inplace=True),
    #5
    nn.Conv2d(384, 256, kernel_size=5, stride=1, padding=2),
    nn.ReLU(inplace=True),
    nn.MaxPool2d(kernel_size=3, stride=2),
    )
    self.avgpool= nn.AvgPool2d(6)
    self.classifier= nn.Sequential(
        nn.Dropout(), nn.Linear(256*6*6, 4096), #128*2*2, 1024
    nn.ReLU(inplace=True), nn.Dropout(),torch.cat((nn.Dropout(),PIs_features)),
    nn.Linear(4096, num_classes))

def forward(self, x):
    x= self.features(x)
    x=x.view(x.size(0), 256*6*6)
    x= self.classifier(x)
    return x

So I wanna implement say 'y' input data with the output of the last dropout layer 'nn.dropout()' in the self.classifier.
Thanks in advance.

Output:
You can do so in the forward definition by simply calling torch.cat((x, y), 1) to concatenate the two feature vectors together.
class AlexNet(nn.Module):
  def __init__(self, num_classes=10):
      super().__init__()
        #1
      self.features= nn.Sequential(
        nn.Conv2d(3, 96, kernel_size=11, stride=4, padding=0),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2),
        #2
        nn.Conv2d(96, 256, kernel_size=5, stride=1, padding=2),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2),
        #3
        nn.Conv2d(256, 384, kernel_size=3, stride=1, padding=1),
        nn.ReLU(inplace=True),
        #4
        nn.Conv2d(384, 384, kernel_size=3, stride=1, padding=1),
        nn.ReLU(inplace=True),
        #5
        nn.Conv2d(384, 256, kernel_size=5, stride=1, padding=2),
        nn.ReLU(inplace=True),
        nn.MaxPool2d(kernel_size=3, stride=2))

      self.avgpool= nn.AvgPool2d(6)
      self.classifier= nn.Sequential(
          nn.Dropout(),
          nn.LazyLinear(4096),
          nn.ReLU(inplace=True),
          nn.Dropout())

      self.fc = nn.LazyLinear(num_classes)

  def forward(self, x, y):
      x = self.features(x)
      x = self.avgpool(x)
      x = x.flatten(1)
      x = torch.cat((x, y), 1)
      x = self.classifier(x)
      return x

Additionally, I have replaced the fully connected nn.Linear layers with LazyLayer. But you can replace them with fixed neurons if you prefer.


Instruction: How to get continuous value from output layer

Input:
I have a code (from here) to classify the MINST digits. The code is working fine. Here they used CrossEntropyLoss and Adam optimizer.
The model code is given below
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(
                in_channels=1,
                out_channels=16,
                kernel_size=5,
                stride=1,
                padding=2,
            ),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(16, 32, 5, 1, 2),
            nn.ReLU(),
            nn.MaxPool2d(2),
        )
        # fully connected layer, output 10 classes
        self.out = nn.Linear(32 * 7 * 7, 10)
        # self.softmax = torch.nn.Softmax(dim=1)
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        # flatten the output of conv2 to (batch_size, 32 * 7 * 7)
        x = x.view(x.size(0), -1)
        output = self.out(x)
        # output = self.softmax(output)
        return output, x    # return x for visualization

The shape of the `b_x` and `b_y` is

torch.Size([100, 1, 28, 28]) torch.Size([100])

Now, I wanted to get continuous value from the output layer. Say, I want the output as alike i.e, 1.0, 0.9, 8.6, 7.0, etc. If the value of the output layer is 1.0 and the label is 1 that means the prediction is perfect. Otherwise, not perfect. More simply, I want to think the MNIST digits as a regression problem.
So, I changed the loss function to MSELoss and optimizer to SGD (the rest of the code remains as the same as the website). But now, I am getting an error
/home/Opps_0/.local/lib/python3.8/site-packages/torch/nn/modules/loss.py:528: UserWarning: Using a target size (torch.Size([100])) that is different to the input size (torch.Size([100, 10])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
Traceback (most recent call last):
  File "/usr/lib/python3.8/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.8/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/home/Opps_0/Desktop/MNIST/src/train.py", line 60, in <module>
    train(NB_EPOCS, model, loaders)
  File "/home/Opps_0/Desktop/MNIST/src/train.py", line 45, in train
    loss = criterion(output, b_y)
  File "/home/Opps_0/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/Opps_0/.local/lib/python3.8/site-packages/torch/nn/modules/loss.py", line 528, in forward
    return F.mse_loss(input, target, reduction=self.reduction)
  File "/home/Opps_0/.local/lib/python3.8/site-packages/torch/nn/functional.py", line 2925, in mse_loss
    expanded_input, expanded_target = torch.broadcast_tensors(input, target)
  File "/home/Opps_0/.local/lib/python3.8/site-packages/torch/functional.py", line 74, in broadcast_tensors
    return _VF.broadcast_tensors(tensors)  # type: ignore
RuntimeError: The size of tensor a (10) must match the size of tensor b (100) at non-singleton dimension 1

Could you tell me what I have to change to get the continuous value for the output layer?

Output:
Assuming your targets are shape as (batch_size,), something along the lines of:
 >>> model = CNN()
 >>> criterion = nn.MSELoss()

 >>> output, _ = model(torch.rand(2, 1, 28, 28))
 >>> b_y = torch.randint(0, 10, (2,))
 tensor([1, 2, 6, 5, 7])

Loss computation with MSELoss will result in:
 >>> loss = criterion(output, b_y)

RuntimeError: The size of tensor a (10) must match the size of tensor b (2) at non-singleton dimension 1.

This means the shape of your target b_y is incorrect, it needs to match output's shaped, i.e. it needs to be a two-dimensional tensor.
Since you are optimizing this task with a regression loss you could encode your target as a sparse vector also known as one-hot encoding. You can do so with ease using the builtin torch.nn.functional.one_hot:
>>> ohe_target = torch.nn.functional.one_hot(b_y, num_classes=10)
tensor([[0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
        [0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 1, 0, 0]])

Now you can compute the loss properly:
>>> criterion(output, ohe_target)
tensor(0.1169, grad_fn=<MseLossBackward>)


Instruction: PyTorch: Inconsistent behavior in torch.arange

Input:
I'm using Google Colab and when I run the following code:
from torch import tensor, arange

print( arange(0.0, 1.2, 0.2) )
print( arange(tensor(0.0), tensor(1.2), tensor(0.2)) )

I get the output:
tensor([0.0000, 0.2000, 0.4000, 0.6000, 0.8000, 1.0000])
tensor([0.0000, 0.2000, 0.4000, 0.6000, 0.8000, 1.0000, 1.2000])

This discrepancy also occurs with arange(0.0,1.1,0.1) but there is no discrepancy with arange(0.0,1.5,0.5).
Why does seemingly similar code give different results, and how can I anticipate when this will occur?

Output:
This is a numerical precision issue :(
By default, Python stores floating point numbers in double precision (aka float64), while PyTorch uses float32 by default.
If you try:
tensor(1.2).item()
tensor(1.2).dtype  # torch.float32

you'll get 1.2000000476837158, and arange with that number will be different from the one with 1.2. In this specific case, if you try:
import torch
from torch import tensor, arange

print(arange(tensor(0.0), tensor(1.2, dtype=torch.float64), tensor(0.2)))

you'll get what you expect, however even float64 will have some precision issue eventually. Anyway, as Python also uses double precision, you won't have this problem converting a scalar back and forth in this case.
I'm not sure if you can anticipate it.


Instruction: Why does torch.nn.Upsample return a junk image?

Input:
When I execute the code segment below, nn.Upsample seems to be completely destroying my image. Am I applying it in the wrong way?
import torch
import imageio
import torch.nn as nn
from matplotlib import pyplot as plt

small = imageio.imread('small.png')                               # shape 200, 390, 4
small_reshaped = small.reshape(4, 200, 390)                       # shape 4, 200, 390
batch = torch.as_tensor(small_reshaped).unsqueeze(0)              # shape 1, 4, 200, 390
ups = nn.Upsample((500, 970))
upsampled_batch = ups(batch)                                      # shape 1, 4, 500, 970
upsampled_small = upsampled_batch[0].reshape(500, 970, 4)         # shape 500, 970, 4
plt.imshow(small)
plt.imshow(upsampled_small)
plt.show()

Before upsampling:

After upsampling:

Original image (small.png):

Output:
Resolved it. Reshaping destroys the image. I should have transposed instead.
See https://discuss.pytorch.org/t/for-beginners-do-not-use-view-or-reshape-to-swap-dimensions-of-tensors/75524 for more details.
A working solution:
...
small_reshaped = small.transpose(2, 0, 1)                          # shape 4, 200, 390
...
upsampled_small = upsampled_batch[0].transpose(0,1).transpose(1,2) # shape 500, 970, 4
...


Instruction: Logistic regression with dropout regularization in PyTorch

Input:
I want to implement a logistic regression with dropout regularization but so far the only working example is the following:
class logit(nn.Module):
    def __init__(self, input_dim = 69, output_dim = 1):
        super(logit, self).__init__()

        # Input Layer (69) -> 1
        self.fc1 = nn.Linear(input_dim, input_dim)
        self.fc2 = nn.Linear(input_dim, 1)

        self.dp = nn.Dropout(p = 0.2)

    # Feed Forward Function
    def forward(self, x):
        x = self.fc1(x)
        x = self.dp(x)
        x = torch.sigmoid(self.fc2(x))

        return x

Now the problem of setting dropout in between layers is that at the end I do not have a logistic regression anymore (correct me if I'm wrong).
What I would like to do is drop out at the input level.

Output:
Actually, you still have a logistic regression with the dropout as it is.
The dropout between fc1 and fc2 will drop some (with p=0.2) of the input_dim features produced by fc1, requiring fc2 to be robust to their absence. This fact doesn't change the logit at the output of your model. Moreover, remember that at test time, (usually) the dropout will be disabled.
Note that you could also apply dropout at the input level:
    def forward(self, x):
        x = self.dp(x)
        x = self.fc1(x)
        x = self.dp(x)
        x = torch.sigmoid(self.fc2(x))

In this case, fc1 would have to be robust to the absence of some of the input features.


Instruction: Using Hugging-face transformer with arguments in pipeline

Input:
I am working on using a transformer. Pipeline to get BERT embeddings to my input. using this without a pipeline i am able to get constant outputs but not with pipeline since I was not able to pass arguments to it.
How can I pass transformer-related arguments for my Pipeline?
# These are BERT and tokenizer definitions
tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")
model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")

inputs = ['hello world']

# Normally I would do something like this to initialize the tokenizer and get the result with constant output
tokens = tokenizer(inputs,padding='max_length', truncation=True, max_length = 500, return_tensors="pt")
model(**tokens)[0].detach().numpy().shape

# using the pipeline
pipeline("feature-extraction", model=model, tokenizer=tokenizer, device=0)

# or other option
tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT",padding='max_length', truncation=True, max_length = 500, return_tensors="pt")
model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")

nlp=pipeline("feature-extraction", model=model, tokenizer=tokenizer, device=0)

# to call the pipeline
nlp("hello world")

I have tried several ways like the options listed above but was not able to get results with constant output size. one can achieve constant output size by setting the tokenizer arguments but have no idea how to give arguments for the pipeline.
any idea?

Output:
The max_length tokenization parameter is not supported per default (i.e. no padding to max_length is applied), but you can create your own class and overwrite this behavior:
from transformers import AutoTokenizer, AutoModel
from transformers import FeatureExtractionPipeline
from transformers.tokenization_utils import TruncationStrategy

tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")
model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")

inputs = ['hello world']

class MyFeatureExtractionPipeline(FeatureExtractionPipeline):
      def _parse_and_tokenize(
        self, inputs, max_length, padding=True, add_special_tokens=True, truncation=TruncationStrategy.DO_NOT_TRUNCATE, **kwargs
    ):
        """
        Parse arguments and tokenize
        """
        # Parse arguments
        if getattr(self.tokenizer, "pad_token", None) is None:
            padding = False
        inputs = self.tokenizer(
            inputs,
            add_special_tokens=add_special_tokens,
            return_tensors=self.framework,
            padding=padding,
            truncation=truncation,
            max_length=max_length
        )
        return inputs

mynlp = MyFeatureExtractionPipeline(model=model, tokenizer=tokenizer)
o = mynlp("hello world", max_length = 500, padding='max_length', truncation=True)

Let us compare the size of the output:
print(len(o))
print(len(o[0]))
print(len(o[0][0]))

Output:
1
500
768

Please note: that this will only work with transformers 4.10.X and previous versions. The team is currently refactoring the pipeline classes and future releases will require different adjustments (i.e. that will not work as soon as the refactored pipelines are released).


Instruction: Torchvision ImageFolder "Could not find any class folder"

Input:
The code below
plastic_train_image_folder = torchvision.datasets.ImageFolder(plastic_dir, transform=transforms)

throws the following error:

Could not find any any class folder in /Users/username/Documents/Jupyter/archive/Garbage classification/Garbage classification/plastic.

Yet, there are files there. The code below prints 482.
list_plastic = os.listdir(plastic_dir)
number_files_plastic = len(list_plastic)
print(number_files_plastic)

Why is this error happening?

Output:
As you can see in the documentation, the ImageFolder class expects images to be within directories, one for each class of interest:

A generic data loader where the images are arranged in this way:
root/dog/xxx.png
root/dog/xxy.png
root/dog/xxz.png

root/cat/123.png
root/cat/nsdf3.png
root/cat/asd932_.png

Your images are probably in the root directory, which is not the way it is expecting, hence the error.


Instruction: Send all parameters and objects of a class to same device in PyTorch

Input:
I have the following dummy code in PyTorch
class inside(nn.Module):
    def __init__(self):
        super(inside, self).__init__()
        self.weight_h = nn.Parameter(SOMETHING GOES HERE) # send to CPU or GPU
        self.weight_v = nn.Conv2d(SOMETHING GOES HERE) # send to CPU or GPU

    def forward(self, x):
        ...
        return x

class main_class(nn.Module):
    def __init__(self):
        super(main_class, self).__init__()
        self.paramA = nn.Conv2d(...)
        self.paramB = nn.Conv2d(...)
        self.in_class = inside()

    def forward(self, x):
        ...
        return x

device = #Define what GPU device to use or CPU

object = main_class()
object = object.to(device)

Suppose in this code the device is GPU2. Then I know that the parameters self.paramA and self.paramB have definitely been loaded to GPU2 and not on CPU or any other GPU. But what can be said of self.weight_h and self.weight_v? Are they guaranteed to be on GPU2 or do I need to explicitly state this for parameters of inside class?
I am using PyTorch 1.8.1 but perhaps suggest a method which is quite general and which will be true for any PyTorch version>=1.0

Output:
I guess when you said this code - the term can be clarified a bit more. There are two things that can be put in GPU. One of the thing is regarding the data. You can keep your data in GPU and things like that.
There is another part to it, the model can be transferred to GPU. In this case, when you do final_model.to(...) then all the modules inside of it as part of the final model would be transferred to GPU.
I differentiated this two because sometimes it is easy to mess these two things up.
So the final answer is,  yes they are guaranteed to be on GPU. (Those inside model weights which are part of the large model).


Instruction: torch.cuda.is_available() returns False why?

Input:
Has anyone encountered this?

I tried updating drivers and reinstalling cuda

Cuda Version: 11.4
GPU: GeForce RTX 3060 Laptop(6gb)
OS: Windows 10 home
torch.version: 1.9.0+cpu

Output:
You are using a PyTorch version compiled for CPU, you should install the appropriate version instead:

Using conda:
conda install pytorch torchvision cudatoolkit=11.1 -c pytorch -c conda-forge

Using pip:
python -m pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html


Instruction: Which version of pysyft works with torch 1.8 or 1.9?

Input:
Which version of pysyft works with torch 1.8 or 1.9?
I have tried to install pysyft using following code
!pip install syft
!pip install syft_proto

This version install old of torch. I want pysyft to install latest version of torch.

Output:
The current latest version of syft declares compatibility with torch <= 1.8.1. Just install the latest syft:
!pip install --upgrade --upgrade-strategy=eager syft


Instruction: Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy() instead

Input:
This is my second question on this problem. Initially, I had the AttributeError error: 'numpy.ndarray' object has no attribute 'log'. Then U12-Forward helped me solve it. But a new problem has arisen.
import torch
import numpy as np
import matplotlib.pyplot as plt

x = torch.tensor([[5., 10.],
                  [1., 2.]], requires_grad=True)
var_history = []
fn_history = []
alpha = 0.001
optimizer = torch.optim.SGD([x], lr=alpha)

def function_parabola(variable):
    return np.prod(np.log(np.log(variable + 7)))

def make_gradient_step(function, variable):
    function_result = function(variable)
    function_result.backward()
    optimizer.step()
    optimizer.zero_grad()

for i in range(500):
    var_history.append(x.data.numpy().copy())
    fn_history.append(function_parabola(x).data.cpu().detach().numpy().copy())
    make_gradient_step(function_parabola, x)
print(x)
def show_contours(objective,
                  x_lims=[-10.0, 10.0],
                  y_lims=[-10.0, 10.0],
                  x_ticks=100,
                  y_ticks=100):
    x_step = (x_lims[1] - x_lims[0]) / x_ticks
    y_step = (y_lims[1] - y_lims[0]) / y_ticks
    X, Y = np.mgrid[x_lims[0]:x_lims[1]:x_step, y_lims[0]:y_lims[1]:y_step]
    res = []
    for x_index in range(X.shape[0]):
        res.append([])
        for y_index in range(X.shape[1]):
            x_val = X[x_index, y_index]
            y_val = Y[x_index, y_index]
            res[-1].append(objective(np.array([[x_val, y_val]]).T))
    res = np.array(res)
    plt.figure(figsize=(7,7))
    plt.contour(X, Y, res, 100)
    plt.xlabel('$x_1$')
    plt.ylabel('$x_2$')
show_contours(function_parabola)
plt.scatter(np.array(var_history)[:,0], np.array(var_history)[:,1], s=10, c='r');
plt.show()

Output:
Modify function_parabola() to operate on PyTorch tensors and leverage PyTorch equivalents of the original numpy operations, like so:
import torch
import numpy as np
import matplotlib.pyplot as plt

x = torch.tensor([[5., 10.],
                  [1., 2.]], requires_grad=True)
var_history = []
fn_history = []
alpha = 0.001
optimizer = torch.optim.SGD([x], lr=alpha)

def function_parabola(variable):
    return (torch.prod(torch.log(torch.log(torch.as_tensor(variable + 7)))))

def make_gradient_step(function, variable):
    function_result = function(variable)
    function_result.backward()
    optimizer.step()
    optimizer.zero_grad()

for i in range(500):
    var_history.append(x.data.numpy().copy())
    fn_history.append(function_parabola(x).data.cpu().detach().numpy())
    make_gradient_step(function_parabola, x)
print(x)
def show_contours(objective,
                  x_lims=[-10.0, 10.0],
                  y_lims=[-10.0, 10.0],
                  x_ticks=100,
                  y_ticks=100):
    x_step = (x_lims[1] - x_lims[0]) / x_ticks
    y_step = (y_lims[1] - y_lims[0]) / y_ticks
    X, Y = np.mgrid[x_lims[0]:x_lims[1]:x_step, y_lims[0]:y_lims[1]:y_step]
    res = []
    for x_index in range(X.shape[0]):
        res.append([])
        for y_index in range(X.shape[1]):
            x_val = X[x_index, y_index]
            y_val = Y[x_index, y_index]
            res[-1].append(objective(np.array([[x_val, y_val]]).T))
    res = np.array(res)
    plt.figure(figsize=(7,7))
    plt.contour(X, Y, res, 100)
    plt.xlabel('$x_1$')
    plt.ylabel('$x_2$')
show_contours(function_parabola)
plt.scatter(np.array(var_history)[:,0], np.array(var_history)[:,1], s=10, c='r');

plt.show()


Instruction: What is equivalent to pytorch lstm num_layers?

Input:
I'm a beginner in PyTorch. From the lstm description, I learned that I can create a stacked lstm with 3 layers by:
layer = torch.nn.LSTM(128, 512, num_layers=3)

Then in the forward function, I can do:
def forward(x, state):
    x, state = layer(x, state)
    return x, (state[0].detach(), state[1].detach())

And I can pass state from batch to batch.
But if I create 3 lstm layers, what is the equivalent to that if I want to implement the same stacked layers myself?
layer1 = torch.nn.LSTM(128, 512, num_layers=1)
layer2 = torch.nn.LSTM(128, 512, num_layers=1)
layer3 = torch.nn.LSTM(128, 512, num_layers=1)

In this case, what should go into the forward function and get the returned state?
I also tried to look at the source code of pytorch lstm, but in the forward function it calls a _VF module which I cannot find where it is defined.

Output:
If you define state as a list of the 3 layers' states, then
def forward(x, state):
    x, s0 = layer1(x, state[0])
    x, s1 = layer2(x, state[1])
    x, s2 = layer3(x, state[2])
    return x, [s0.detach(), s1.detach(), s2.detach()]


Instruction: Timing Neural Network Inference Standards

Input:
I need to measure neural network inference times for a project. I want my results presented to be aligned with the standard practices for measuring this in academic papers.
What I have managed to figure out is that we first want to warm up the GPU with a few inferences before the timing, and I need to use the torch provided timing feature (instead of Python's time.time()).
My questions are as follows:

Is it standard to time with a batch size of 1, or with the best batch size for that hardware?
Am I only timing the neural network inference, or am I also timing the moving of data to the GPU, as well as data transformations that precede inference?
How many iterations would be reasonable to time to get a good average inference time?

Any advice would be greatly appreciated. Thank you.

Output:
If you're concerned with inference time, batch size should be something to optimize for in the first place. Not all operations in a NN will be affected in the same way by a change in batch size (you could have not change thanks to parallelization, or linear change if all kernels are busy for instance). If you need to compare between models I'd optimize per model. If you don't want to do that then I'd use the train-time batch size. I think it would be unlikely that in production you'd have a batch size of 1, except if it does not fit in memory.

You should time both. If you're comparing models, data loading and transforms should not impact your decision but in a production environment it will matter a lot. So report both numbers, in some settings, scaling up the data-loading or the model may be easier than the other.

I would say around a 100. It's just a rule of thumb. You want your numbers to be statistically significant. You can also report the std in addition to the average, or even plot the distribution (percentiles or histograms or else)

You can also compare performance loss vs inference time gain when using half float types for your data and model weights.


Instruction: What is the Best way to define Adam Optimizer in PyTorch?

Input:
For most PyTorch codes we use the following definition of Adam optimizer,
optim = torch.optim.Adam(model.parameters(), lr=cfg['lr'], weight_decay=cfg['weight_decay'])

However, after repeated trials, I found that the following definition of Adam gives 1.5 dB higher PSNR which is huge.
optim = torch.optim.Adam(
            [
                {'params': get_parameters(model, bias=False)},
                {'params': get_parameters(model, bias=True), 'lr': cfg['lr'] * 2, 'weight_decay': 0},
            ],
            lr=cfg['lr'],
            weight_decay=cfg['weight_decay'])

The Model is a usual U-net with parameters defined in init and forward action as in any other PyTorch model.
The get_parameters is defined as below.
def get_parameters(model, bias=False):
    for k, m in model._modules.items():
        print("get_parameters", k, type(m), type(m).__name__, bias)
        if bias:
            if isinstance(m, nn.Conv2d):
                yield m.bias
        else:
            if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
                yield m.weight

Could someone explain why the latter definition is better than the previous one?

Output:
In the second method, different configurations are being provided to update weights and biases. This is being done using per-parameter options for the optimizer.
optim = torch.optim.Adam(
            [
                {'params': get_parameters(model, bias=False)},
                {'params': get_parameters(model, bias=True), 'lr': cfg['lr'] * 2, 'weight_decay': 0},
            ],
            lr=cfg['lr'],
            weight_decay=cfg['weight_decay'])

As per this, the learning rate for biases is 2 times that of weights, and weight decay is 0.
Now, the reason why it's being done could be the network not learning properly. Read more Why is the learning rate for the bias usually twice as large as the the LR for the weights?