Instruction: Runtime error: CUDA out of memory by the end of training and doesn’t save model; pytorch

Input:
I'm not so experienced in Data Science and pytorch and I have problems with implementing at least anything here(currently I'm making a NN for segmentation tasks). There is some kind of memory problem, although it doesn't meen anything - every epoch takes a lot less memory than it is in the risen
import torch
from torch import nn
from torch.autograd import Variable
from torch.nn import Linear, ReLU6, CrossEntropyLoss, Sequential, Conv2d, MaxPool2d, Module, Softmax, Softplus ,BatchNorm2d, Dropout, ConvTranspose2d
import torch.nn.functional as F
from torch.nn import LeakyReLU,Tanh
from torch.optim import Adam, SGD
import numpy as np
import cv2 as cv
def train(epoch,model,criterion, x_train, y_train, loss_val):
    model.train()
    tr_loss = 0
    # getting the training set
    x_train, y_train = Variable(x_train), Variable(y_train)
    # converting the data into GPU format

    # clearing the Gradients of the model parameters
    optimizer.zero_grad()

    # prediction for training and validation set
    output_train = model(x_train)
    # computing the training and validation loss
    loss_train = criterion(output_train, y_train)
    train_losses.append(loss_train)
    # computing the updated weights of all the model parameters
    loss_train.backward()
    optimizer.step()
    tr_loss = loss_train.item()
    return loss_train
        # printing the validation loss

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 96, (3,3), padding=1)
        self.conv11= nn.Conv2d(96, 96, (3,3), padding=1)
        self.conv12= nn.Conv2d(96, 96, (3,3), padding=1)
        self.pool  = nn.MaxPool2d((2,2), 2)
        self.conv2 = nn.Conv2d(96, 192, (3,3), padding=1)
        self.conv21 = nn.Conv2d(192, 192, (3,3), padding=1)
        self.conv22 = nn.Conv2d(192, 192, (3,3), padding=1)
        self.b = BatchNorm2d(96)
        self.b1 = BatchNorm2d(192)
        self.b2 = BatchNorm2d(384)
        self.conv3 = nn.Conv2d(192,384,(3,3), padding=1)
        self.conv31= nn.Conv2d(384,384,(3,3), padding=1)
        self.conv32= nn.Conv2d(384,384,(3,3), padding=1)
        self.lin1   = nn.Linear(384*16*16, 256*2*2, 1)
        self.lin2   = nn.Linear(256*2*2, 16*16, 1)
        self.uppool = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)
        self.upconv1= nn.ConvTranspose2d(385,192,(3,3), padding=1)
        self.upconv11=nn.ConvTranspose2d(192,32,(3,3), padding=1)
        self.upconv12=nn.ConvTranspose2d(32,1,(3,3), padding=1)
        self.upconv2= nn.ConvTranspose2d(193,96,(3,3), padding=1)
        self.upconv21= nn.ConvTranspose2d(96,16,(3,3), padding=1)
        self.upconv22= nn.ConvTranspose2d(16,1,(3,3), padding=1)
        self.upconv3= nn.ConvTranspose2d(97,16,(3,3), padding=1)
        self.upconv4= nn.ConvTranspose2d(16,8,(3,3), padding=1)
        self.upconv6= nn.ConvTranspose2d(8,1,(3,3), padding=1)
    def forward(self, x):
        m=Tanh()
        x1=self.b(m(self.conv12(m(self.conv11(m(self.conv1(x)))))))
        x = self.pool(x1)
        x2=self.b1(m(self.conv22(m(self.conv21(m(self.conv2(x)))))))
        x = self.pool(x2)
        x3=self.b2(m(self.conv32(m(self.conv31(m(self.conv3(x)))))))
        x=self.pool(x3)
        x = x.view(-1, 16*16*384)
        x = m(self.lin1(x))
        x = m(self.lin2(x))
        x = x.view(1, 1, 16, 16)
        x=torch.cat((x,self.pool(x3)),1)
        x = self.uppool(m(self.upconv12(m(self.upconv11(m(self.upconv1(x)))))))

        x=torch.cat((x,self.pool(x2)),1)
        x = self.uppool(m(self.upconv22(m(self.upconv21(m(self.upconv2(x)))))))

        x=torch.cat((x,self.pool(x1)),1)
        x = (self.uppool(m(self.upconv3(x))))
        x = (m(self.upconv4(x)))
        l=Softplus()
        x= l(self.upconv6(x))
        return x
train_data=[]
for path in range(1000):
    n="".join(["0" for i in range(5-len(str(path)))])+str(path)
    paths="00000\\"+n+".png"
    train_data.append(cv.imread(paths))
for path in range(2000,3000):
    n="".join(["0" for i in range(5-len(str(path)))])+str(path)
    paths="02000\\"+n+".png"
    train_data.append(cv.imread(paths))
train_output=[]
for path in range(1,2001):
    n="outputs\\"+str(path)+".jpg"
    train_output.append(cv.imread(n))
data=torch.from_numpy((np.array(train_data,dtype=float).reshape(2000,3,128,128)/255)).reshape(2000,3,128,128)
data_cuda=torch.tensor(data.to('cuda'), dtype=torch.float32)

output=torch.from_numpy(np.array(train_output,dtype=float).reshape(2000,3,128,128))[:,2].view(2000,1,128,128)*2
output_cuda=torch.tensor(output.to('cuda'),dtype=torch.float32)
model=Net()
optimizer = Adam(model.parameters(), lr=0.1)
criterion = nn.BCEWithLogitsLoss()
if torch.cuda.is_available():
    model = model.cuda()
    criterion = criterion.cuda()
print(model)
epochs=3
n_epochs = 1
train_losses = []
val_losses = []
for epoch in range(n_epochs):
    loss_train=0
    for i in range(data.shape[0]):
        loss_train1=train(epoch,model,criterion,data_cuda[i].reshape(1,3,128,128),output_cuda[i].reshape(1,1,128,128),train_losses)
        loss_train+=loss_train1
    print('Epoch : ',epoch+1, '\t', 'loss :', loss_train/data.shape[0])
with torch.no_grad():
    torch.save(model.state_dict(), "C:\\Users\\jugof\\Desktop\\Python\\pytorch_models")
    a=np.array(model(data_cuda).to('cpu').numpy())*255
    cv.imshow('',a.reshape(128,128))
    cv.waitKey(0)"""

Here is the error:

PS C:\Users\jugof\Desktop\Python> & C:/Users/jugof/anaconda3/python.exe c:/Users/jugof/Desktop/Python/3d_visual_effect1.py
c:/Users/jugof/Desktop/Python/3d_visual_effect1.py:98: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
data_cuda=torch.tensor(data.to('cuda'), dtype=torch.float32)
c:/Users/jugof/Desktop/Python/3d_visual_effect1.py:101: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
output_cuda=torch.tensor(output.to('cuda'),dtype=torch.float32)
Epoch :  1       loss : tensor(0.6933, device='cuda:0', grad_fn=)
Traceback (most recent call last):
File "c:/Users/jugof/Desktop/Python/3d_visual_effect1.py", line 120, in
a=np.array(model(data_cuda).to('cpu').numpy())*255
File "C:\Users\jugof\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 889, in _call_impl
result = self.forward(*input, **kwargs)
File "c:/Users/jugof/Desktop/Python/3d_visual_effect1.py", line 62, in forward
x1=self.b(m(self.conv12(m(self.conv11(m(self.conv1(x)))))))
File "C:\Users\jugof\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 889, in _call_impl
result = self.forward(input, **kwargs)
File "C:\Users\jugof\anaconda3\lib\site-packages\torch\nn\modules\conv.py", line 399, in forward
return self._conv_forward(input, self.weight, self.bias)
File "C:\Users\jugof\anaconda3\lib\site-packages\torch\nn\modules\conv.py", line 395, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
RuntimeError: CUDA out of memory. Tried to allocate 11.72 GiB (GPU 0; 6.00 GiB total capacity; 2.07 GiB already allocated; 1.55 GiB free; 2.62 GiB reserved in total by PyTorch)
I feed a numpy array (an image) of 128128 shape and recieve another of the same shape, it's a segmentation model(again)

I was using Flickr-Faces-HQ Dataset (FFHQ) and used downsampled 128*128 labels - I used 00000, 01000 and 02000 files and masks were recieved by opencv haarscascades_eye

Output:
The problem is your loss_train list, which stores all losses from the beginning of your experiment. If the losses you put in were mere float, that would not be an issue, but because of your not returning a float in the train function, you are actually storing loss tensors, with all the computational graph embedded in them. Indeed, a tensor keeps pointers of all tensors that were involved in its computation, and as long as a pointer exist, the allocated memory cannot be freed.
So basically, you keep all tensors from all epochs and prevent pytorch from cleaning them ; it's like a (deliberate) memory leak
You can very easily monitor this type of issue by running nvidia-smi -l 1 after having started your experiment. You will watch your memory usage grow linearly until your GPU runs out of memory (`nvidia-smi is a good tool to use when doing stuff on your GPU).
To prevent this from happening, simply replace the last line of the train function with return loss_train.item(), and the memory issue will vanish


Instruction: In Pytorch, quantity.backward() computes the gradient of quantity wrt which of the parameters?

Input:
The backward method computes the gradient wrt to which parameters? All of the params with requires_grad having True value?
Interestingly, in Pytorch

computing gradients

and

loading the optimizer that updates parameters based on gradients

need different informations about the identity of parameters of interest to be able to work.
The first one seem to know which parameters to compute the gradient for.
The second one needs the parameters to be mentioned to it. See the code below.
quantity.backward()
optim = torch.SGD(model.parameters())
optim.step()

How is that?
Why backward does not need the model.parameters()?
Would it not be more efficient to mention the specific subset of parameters?

Output:
Computing quantity requires constructing a 2-sorted graph with nodes being either tensors or differentiable operations on tensors (a so-called computational graph). Under the hood, pytorch keeps track of this graph for you. When you call quantity.backward(), you're asking pytorch to perform an inverse traversal of the graph, from the output to the inputs, using the derivative of each operation encountered rather the operation itself. Leaf tensors that are flagged as requiring gradients accumulate the gradients computed by backward.
An optimizer is a different story: it simply implements an optimization strategy on a set of parameters, hence it needs to know which parameters you want it to be optimizing. So quantity.backward() computes gradients, optim.step() uses these gradients to perform on a optimization step, updating the parameters contained in model.
As for efficiency, I don't see any argument in favor of specifying parameters in the backward pass (what would the semantics of that be?).
If what you'd want is to avoid traversal of parts of the graph in backward mode, pytorch will do it automagically for you if you remember:

you can mark leaf tensors as not requiring grad
a non-leaf tensor -- the output of some operation f(x1,...xN) -- requires grad if at least one of x1...xN requires grad
a tensor that doesn't require grad blocks backward traversal, ensuring no unnecessary computation


Instruction: How can I handle this datasets to create a datasetDict?

Input:
I'm trying to build a datasetDictionary object to train a QA model on PyTorch. I have these two different datasets:
test_dataset

Dataset({
    features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
    num_rows: 21489
})

and
train_dataset

Dataset({
    features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
    num_rows: 54159
})

In the dataset's documentation I didn't find anything. I'm quite a noob, thus the solution may be really easy. What I wish to obtain is something like this:
dataset

DatasetDict({
    train: Dataset({
        features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
        num_rows: 54159
    })
    test: Dataset({
        features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
        num_rows: 21489
    })
})

I really don't find how to use two datasets to create a dataserDict or how to set the keys. Moreover, I wish to "cut" the train set in two: train and validation sets, but also this passage is hard for me to handle. The final result should be something like this:
dataset

DatasetDict({
    train: Dataset({
        features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
        num_rows: 54159 - x
    })
    validation: Dataset({
        features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
        num_rows: x
    })
    test: Dataset({
        features: ['answer_text', 'answer_start', 'title', 'context', 'question', 'answers', 'id'],
        num_rows: 21489
    })
})

Thank you in advance and pardon me for being a noob :)

Output:
to get the validation dataset, you can do like this:
train_dataset, validation_dataset= train_dataset.train_test_split(test_size=0.1).values()

This function will divide 10% of the train dataset into the validation dataset.
and to obtain "DatasetDict", you can do like this:
import datasets
dd = datasets.DatasetDict({"train":train_dataset,"test":test_dataset})


Instruction: i'm confused with CoordConv

Input:
i read a paper which written by uber lab
https://medium.com/@Cambridge_Spark/coordconv-layer-deep-learning-e02d728c2311
they create a network named Coordconv,and in this coordconv they not only add two layer of meshgrid but also with a simple conv net.

it said through this way they add positional info to every pixel points?
2.so that after conv the pixel points still remain in same place as in original image?
and this is also working to add two layers of meshgrid to freature maps which draw from neural network?
how could Meshgrid help add positional info to the image?
Does this just simply added two layers which are the same size as the original image but is in[-1,1] meshgrid to original input image?

a big THANKS in advance!

Output:
About CoordConv
Here is the original paper which proposed the CoordConv layer: CoordConv paper.
I will try to convey my instinctive undersanding of this operation.
How AddCoords works
The way the information is added is by stacking (concatenating, to be more accurate) two new 2D tensors to the data. Those two channels are not multiplied together, therefore there is no meshgrid involved in this process.
Say we are at a specific layer of the network. The last convolution step produced 4 2D-tensors of shape 8x8, each of which is the result of the previous convolution by a filter (thus we had 4 kernels in the previous step). They are in reality stacked in a single tensor of size bs * 8 * 8 * 4 where bs is the batch size, but let's ignore the batch size from now.
The AddCoords method will create two other 2D tensors:
xx_channel:
[[0, 0, 0, 0, 0, 0, 0, 0],
 [1, 1, 1, 1, 1, 1, 1, 1],
 [2, 2, 2, 2, 2, 2, 2, 2],
 [3, 3, 3, 3, 3, 3, 3, 3],
 [4, 4, 4, 4, 4, 4, 4, 4],
 [5, 5, 5, 5, 5, 5, 5, 5],
 [6, 6, 6, 6, 6, 6, 6, 6],
 [7, 7, 7, 7, 7, 7, 7, 7]]

and yy_channel:
[[0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7],
 [0, 1, 2, 3, 4, 5, 6, 7]]

Those are the results of the matmuls of the tf.range by the tf.ones.
They will then be scaled to fit in the range [-1, 1] and casted to tensorflow.float32 type:
xx_channel:
[[-1.        , -1.        , -1.        , -1.        , -1.        , -1.        , -1.        , -1.        ],
 [-0.71428571, -0.71428571, -0.71428571, -0.71428571, -0.71428571, -0.71428571, -0.71428571, -0.71428571],
 [-0.42857143, -0.42857143, -0.42857143, -0.42857143, -0.42857143, -0.42857143, -0.42857143, -0.42857143],
 [-0.14285714, -0.14285714, -0.14285714, -0.14285714, -0.14285714, -0.14285714, -0.14285714, -0.14285714],
 [ 0.14285714,  0.14285714,  0.14285714,  0.14285714,  0.14285714,  0.14285714,  0.14285714,  0.14285714],
 [ 0.42857143,  0.42857143,  0.42857143,  0.42857143,  0.42857143,  0.42857143,  0.42857143,  0.42857143],
 [ 0.71428571,  0.71428571,  0.71428571,  0.71428571,  0.71428571,  0.71428571,  0.71428571,  0.71428571],
 [ 1.        ,  1.        ,  1.        ,  1.        ,  1.        ,  1.        ,  1.        ,  1.        ]]

yy_channel:
[[-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.],
 [-1., -0.71428571, -0.42857143, -0.14285714,  0.14285714, 0.42857143,  0.71428571,  1.]]

They will then be concatenated to the other 2D-tensors along the last dimension ("-1"), ending up with a 3D-tensor with shape 8 * 8 * 6(again, the dimension of the batch size is ignored in my explanation).
Those two generated channels are what the authors in the paper call coordinate informations. The method literally adds the coordinates of each 2D position : the y-coord and the x-coord.
In our example, let's consider the values of an input tensor at position [4, 5],  meaning the values along the last dimension (size 4), which is accessible like this : input_tensor[4, 5, :]. It may return something like this :
input_tensor[4, 5, :]
# > [0.75261177, 0.62114716, 0.76845441, 0.44747785]

After AddCoords, it becomes:
ret[4, 5, :]
# > [0.75261177, 0.62114716, 0.76845441, 0.44747785, 0.14285714, 0.42857143]

... where 0.14285714 is the scaled value of 4 ie its y-coord and 0.42857143 is the scaled value of 5 ie its x-coord. The information about coordinates is now contained inside the resulting tensor, which is returned by the AddCoords method.
The CoordConv
It's a designed layer that applies AddCoords to the input and feeds the resulting tensor to a classic Conv2D layer. As such, it can be added to a neural network, as you would do with a Conv2D layer.
That's what the authors did, when experimenting with GANs for example, where they substitued Conv2D with CoordConv (which, again, includes a Conv2D).
Let me know if that answers your questions and/or correct any misconceptions.
What does it imply for the neural network ?
More trainable parameters...
Let's give a bit more context to our previous example.
In our previous example, the last layer yielded a tensor with shape 8 x 8 x 4. Let's say we want the next convolution layer to yield 16 output filters, from a convolution window of 3 * 3.

You can see this link to get what convolution does mathematically ,
chapter 2.1 . You can get a basic understanding of what the
convolution operation yields thanks to this visualizer. Just keep
in mind both links show a single kernel and a single channel input
matrix.

If we don't add the coordinate tensors, the convolution to come will have 16 kernels with shape 3 x 3 x 4 each.
If we do apply AddCoords, we will feed a tensor with shape 8 x 8 x 6 instead, and our 16 kernels will each have the shape 3 x 3 x 6.

You can think of those kernels as neurons. Each neuron has 3 x 3 x 4 == 36  weights (Conv2D) or 3 x 3 x 6 == 54 weights (AddCoords+Conv2D, or CoordConv). Their weights will be updated during the learning process. Knowing this, it should appear evident that the coordinates channels of CoordConv implies new and specific weights to each kernel of the convolution layer. That's how the neural network takes into consideration these coordinates.
... implied in similar training processes
If you haven't been experimenting with Machine Learning, the supervised learning process of a neural network might be quite complex to comprehend, but it's more general and could be resumed (oversimplified) as:

We calculate the error, which is a mathematical way to describe how
far the prediction is from the ground truth. Then we update (add)
each parameter (or weight) in the network, layer after layer from the
output layer to the input one, by a value that represents its
implication in this error and the direction it should take to decrease the error. This process is called "backpropagation of
the error".


Instruction: pytorch isn't running on gpu while true

Input:
I want to train on my local gpu but it's only running on cpu while torch.cuda.is_available() is actually true and i can see my gpu but it runs only on cpu , so how to fix it
my CNN model:
import torch.nn as nn
import torch.nn.functional as F
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
# define the CNN architecture

class Net(nn.Module):
    ### TODO: choose an architecture, and complete the class
    def __init__(self):
        super(Net, self).__init__()
        ## Define layers of a CNN
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        # convolutional layer (sees 16x16x16 tensor)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        # convolutional layer (sees 8x8x32 tensor)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)

        # max pooling layer
        self.pool = nn.MaxPool2d(2, 2)
        # linear layer (64 * 4 * 4 -> 500)
        self.fc1 = nn.Linear(64 * 28 * 28, 500)
        # linear layer (500 -> 10)
        self.fc2 = nn.Linear(500, 133)
        # dropout layer (p=0.25)
        self.dropout = nn.Dropout(0.25)

    def forward(self, x):
        ## Define forward behavior
        x = self.pool(F.relu(self.conv1(x)))
        #print(x.shape)
        x = self.pool(F.relu(self.conv2(x)))
        #print(x.shape)
        x = self.pool(F.relu(self.conv3(x)))
        #print(x.shape)

        #print(x.shape)
        # flatten image input
        x = x.view(-1, 64 * 28 * 28)
        # add dropout layer
        x = self.dropout(x)
        # add 1st hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        # add dropout layer
        x = self.dropout(x)
        # add 2nd hidden layer, with relu activation function
        x = self.fc2(x)

        return x

#-#-# You so NOT have to modify the code below this line. #-#-#

# instantiate the CNN
model_scratch = Net()

# move tensors to GPU if CUDA is available
if use_cuda:
    print("TRUE")
    model_scratch = model_scratch.cuda()

train function :
def train(n_epochs, loaders, model, optimizer, criterion, use_cuda, save_path):
    """returns trained model"""
    # initialize tracker for minimum validation loss
    valid_loss_min = np.Inf

    loaders_scratch = {'train': train_loader,'valid': valid_loader,'test': test_loader}

    for epoch in range(1, n_epochs+1):
        # initialize variables to monitor training and validation loss
        train_loss = 0.0
        valid_loss = 0.0

        ###################
        # train the model #
        ###################
        model.train()
        for batch_idx, (data, target) in enumerate(loaders['train']):
            # move to GPU
            if use_cuda:
                data, target = data.cuda(), target.cuda()
            ## find the loss and update the model parameters accordingly
            ## record the average training loss, using something like
            ## train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.data - train_loss))
            # clear the gradients of all optimized variables
            optimizer.zero_grad()
            # forward pass: compute predicted outputs by passing inputs to the model
            output = model(data)
            # calculate the batch loss
            loss = criterion(output, target)
            # backward pass: compute gradient of the loss with respect to model parameters
            loss.backward()
            # perform a single optimization step (parameter update)
            optimizer.step()
            # update training loss
            train_loss += loss.item()*data.size(0)

        ######################
        # validate the model #
        ######################
        model.eval()
        for batch_idx, (data, target) in enumerate(loaders['valid']):
            # move to GPU
            if use_cuda:
                data, target = data.cuda(), target.cuda()
            ## update the average validation loss
            output = model(data)
            # calculate the batch loss
            loss = criterion(output, target)
            # update average validation loss
            valid_loss += loss.item()*data.size(0)

        # calculate average losses
        train_loss = train_loss/len(train_loader.dataset)
        valid_loss = valid_loss/len(valid_loader.dataset)

        # print training/validation statistics
        print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(
            epoch,
            train_loss,
            valid_loss
            ))

        ## TODO: save the model if validation loss has decreased
        if valid_loss <= valid_loss_min:
            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
            valid_loss_min,
            valid_loss))
            torch.save(model.state_dict(), save_path)
            valid_loss_min = valid_loss

    # return trained model
    return model

# train the model
loaders_scratch = {'train': train_loader,'valid': valid_loader,'test': test_loader}
model_scratch = train(100, loaders_scratch, model_scratch, optimizer_scratch,
                      criterion_scratch, use_cuda, 'model_scratch.pt')

# load the model that got the best validation accuracy
model_scratch.load_state_dict(torch.load('model_scratch.pt'))

while i am getting "TRUE" in torch.cuda.is_available() but still not running on GPU
i am only running on CPU
the below picture shows that i am running on cpu with 62%

Output:
To utilize cuda in pytorch you have to specify that you want to run your code on gpu device.
a line of code like:
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")

will determine whether you have cuda available and if so, you will have it as your device.
later in the code you have to pass your tensors and model to this device:
net = net.to(device)

and do the same for your other tensors that need to go to gpu, like test and training values.


Instruction: Understanding the order when reshaping a tensor

Input:
For a tensor:
x = torch.tensor([
    [
        [[0.4495, 0.2356],
          [0.4069, 0.2361],
          [0.4224, 0.2362]],

         [[0.4357, 0.6762],
          [0.4370, 0.6779],
          [0.4406, 0.6663]]
    ],
    [
        [[0.5796, 0.4047],
          [0.5655, 0.4080],
          [0.5431, 0.4035]],

         [[0.5338, 0.6255],
          [0.5335, 0.6266],
          [0.5204, 0.6396]]
    ]
])

Firstly would like to split it into 2 (x.shape[0]) tensors then concat them. Here, i dont really have to actually split it as long as i get the correct output, but it makes a lot more sense to me visually to split it then concat them back together.
For example:
# the shape of the splits are always the same
split1 = torch.tensor([
    [[0.4495, 0.2356],
    [0.4069, 0.2361],
    [0.4224, 0.2362]],

    [[0.4357, 0.6762],
    [0.4370, 0.6779],
    [0.4406, 0.6663]]
])
split2 = torch.tensor([
    [[0.5796, 0.4047],
    [0.5655, 0.4080],
    [0.5431, 0.4035]],

    [[0.5338, 0.6255],
    [0.5335, 0.6266],
    [0.5204, 0.6396]]
])

split1 = torch.cat((split1[0], split1[1]), dim=1)
split2 = torch.cat((split2[0], split2[1]), dim=1)
what_i_want = torch.cat((split1, split2), dim=0).reshape(x.shape[0], split1.shape[0], split1.shape[1])

For the above result, i thought directly reshaping x.reshape([2, 3, 4]) would work, it resulted in the correct dimension but incorrect result.
In general i am:

not sure how to split the tensor into x.shape[0] tensors.
confused about how reshape works. Most of the time i am able to get the dimension right, but the order of the numbers are always incorrect.

Thank you

Output:
The order of the elements in memory in python, pytorch, numpy, c++ etc. are in row-major ordering:
[ first, second
  third, forth  ]

While in matlab, fortran, etc. the order is column major:
[ first,  third
  second, fourth ]

For higher dimensional tensors, this means elements are ordered from the last dimension to the first.
You can easily visualize it using torch.arange followed by .view:
a = torch.arange(24)
a.view(2,3,4)

Results with

tensor([[[ 0,  1,  2,  3],
    [ 4,  5,  6,  7],
    [ 8,  9, 10, 11]],

   [[12, 13, 14, 15],
    [16, 17, 18, 19],
    [20, 21, 22, 23]]])

As you can see the elements are ordered first by row (last dimension), then by column, and finally by the first dimension.
When you reshape a tensor, you do not change the underlying order of the elements, only the shape of the tensor. However, if you permute a tensor - you change the underlying order of the elements.
Look at the difference between a.view(3,2,4) and a.permute(0,1,2) - the shape of the resulting two tensors is the same, but not the ordering of elements:
In []: a.view(3,2,4)
Out[]:
tensor([[[ 0,  1,  2,  3],
         [ 4,  5,  6,  7]],

        [[ 8,  9, 10, 11],
         [12, 13, 14, 15]],

        [[16, 17, 18, 19],
         [20, 21, 22, 23]]])

In []: a.permute(1,0,2)
Out[]:
tensor([[[ 0,  1,  2,  3],
         [12, 13, 14, 15]],

        [[ 4,  5,  6,  7],
         [16, 17, 18, 19]],

        [[ 8,  9, 10, 11],
         [20, 21, 22, 23]]])


Instruction: Can I add new training pictures to my object detection model without re-running the whole training again?

Input: I used yolov5 to train an object detection model. is it possible to add more annotated images after i have already trained the original model or must i restart the whole training with the new set of images?

Output: You are asking about continual learning - this is a very active field of research, and there is no single solution/method to tackle it. You'll have to do more research to find the right approach for your specific settings.


Instruction: Pytorch mixed precision learning, torch.cuda.amp running slower than normal

Input:
I am trying to infer results out of a normal resnet18 model present in torchvision.models attribute. The model is simply trained without any mixed precision learning, purely on FP32.
However, I want to get faster results while inferencing, so I enabled torch.cuda.amp.autocast() function only while running a test inference case.
The code for the same is given below -
model = torchvision.models.resnet18()
model = model.to(device) # Pushing to GPU

# Train the model normally

Without amp -
tensor = torch.rand(1,3,32,32).to(device) # Random tensor for testing
with torch.no_grad():
  model.eval()
  start = torch.cuda.Event(enable_timing=True)
  end = torch.cuda.Event(enable_timing=True)
  model(tensor) # warmup
  model(tensor) # warmpup
  start.record()
  for i in range(20): # total time over 20 iterations
    model(tensor)
  end.record()
  torch.cuda.synchronize()

  print('execution time in milliseconds: {}'. format(start.elapsed_time(end)/20))

  execution time in milliseconds: 5.264944076538086

With amp -
tensor = torch.rand(1,3,32,32).to(device)
with torch.no_grad():
  model.eval()
  start = torch.cuda.Event(enable_timing=True)
  end = torch.cuda.Event(enable_timing=True)
  model(tensor)
  model(tensor)

  start.record()
  with torch.cuda.amp.autocast(): # autocast initialized
    for i in range(20):
      model(tensor)
  end.record()
  torch.cuda.synchronize()

  print('execution time in milliseconds: {}'. format(start.elapsed_time(end)/20))

  execution time in milliseconds: 10.619884490966797

Clearly, the autocast() enabled code is taking double the time. Even, with larger models like resnet50, the timing variation is approximately the same.
Can someone help me out regarding this ? I am running this example on Google Colab and below are the specifications of the GPU
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.27       Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   43C    P0    28W / 250W |      0MiB / 16280MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

torch.version.cuda == 10.1
torch.__version__  == 1.8.1+cu101

Output:
It's most likely because of the GPU you're using - P100, which has 3584 CUDA cores but 0 tensor cores -- the latter of which typically play the main role in mixed precision speedup. You may want to take a quick look at the "Hardware Comparison" section on this article.
If you're stuck to using Colab, the only way I can foresee a possible speedup is if you get assigned a T4, which has tensor cores.
Furthermore, it seems like you're using only a single image / a batch size of 1. If you get a T4, try re-running your benchmarks also using a larger batch size, like maybe 32-64-128-256 etc. You should be able to notice much more visible improvements when you parallelize over batches.


Instruction: How to convert this tensor flow code into pytorch code?

Input:
I am trying to implement an Image Denoising Gan which is written in tensorflow to pytorch and I am unable to understand what is tf.variable_scope and tf.Variable similar in pytorch. please help.
def conv_layer(input_image, ksize, in_channels, out_channels, stride, scope_name, activation_function=lrelu, reuse=False):
    with tf.variable_scope(scope_name):
        filter = tf.Variable(tf.random_normal([ksize, ksize, in_channels, out_channels], stddev=0.03))
        output = tf.nn.conv2d(input_image, filter, strides=[1, stride, stride, 1], padding='SAME')
        output = slim.batch_norm(output)
        if activation_function:
            output = activation_function(output)
        return output, filter

def residual_layer(input_image, ksize, in_channels, out_channels, stride, scope_name):
    with tf.variable_scope(scope_name):
        output, filter = conv_layer(input_image, ksize, in_channels, out_channels, stride, scope_name+"_conv1")
        output, filter = conv_layer(output, ksize, out_channels, out_channels, stride, scope_name+"_conv2")
        output = tf.add(output, tf.identity(input_image))
        return output, filter

def transpose_deconvolution_layer(input_tensor, used_weights, new_shape, stride, scope_name):
    with tf.varaible_scope(scope_name):
        output = tf.nn.conv2d_transpose(input_tensor, used_weights, output_shape=new_shape, strides=[1, stride, stride, 1], padding='SAME')
        output = tf.nn.relu(output)
        return output

def resize_deconvolution_layer(input_tensor, new_shape, scope_name):
    with tf.variable_scope(scope_name):
        output = tf.image.resize_images(input_tensor, (new_shape[1], new_shape[2]), method=1)
        output, unused_weights = conv_layer(output, 3, new_shape[3]*2, new_shape[3], 1, scope_name+"_deconv")
        return output

Output:
You can replace tf.Variable with torch.tensor, torch.tensor can hold gradients all the same.
In torch, you also don't create a graph and then access things in there by name via some scope. You would just create the tensor and then can access it directly. The output variable there would just be accessible to you do with it however you want and to reuse however you see fit.
In fact, if you're code isn't directly using this variable scope then you can likely just ignore it. Often the variable scopes are just to give convenient names to thing if you were ever to inspect the graph.


Instruction: Simulating many agents in PyTorch using multiprocessing

Input:
I want to simulate multiple reinforcement learning agents that are coded using Pytorch. The agents do not share any data dynamically, so I expect that the task should be "embarassingly parallel". I need a lot of simulations (I want to see what is the distribution my agents converge to) so I hope to speed it up using multiprocessing.
I have a model class that stores all the parameters of my agents (which are the same across agents) and the environment. I can simulate N agents over T periods using
model.simulate(N = 10, T = 50)

My class would then run simulation loops and store all networks and simulation histories. I am very new to parallel programming, and I (naively) try the following:
import torch.multiprocessing as mp

num_processes = 6
processes = []
for _ in range(num_processes):
    p = mp.Process(target=model.simulate(N = 10, T = 50), args= ())
    p.start()
    processes.append(p)
for p in processes:
    p.join()

For now I do not even try to store results, I just want to see some speed-up. But the time it takes to run the code above is roughly the same as when I simply run a loop and do 6 simulations consequently:
for _ in range(num_processes):
    model.simulate(N = 10, T = 50)

I also tried to make processes for different instances of the model class, but it did not help.

Output:
It looks like your problem is in this line
p = mp.Process(target=model.simulate(N = 10, T = 50), args= ())

The part model.simulate(N = 10, T = 50) is executed first, then the result (I'm assuming None if there is no return from this method) is passed to the mp.Process as the target parameter. So you are doing all the computation sequentially, and not performing it on the new processes.
What you need to do instead is to pass the simulate function (without executing it) and provide the args separately.
i.e. something like...
p = mp.Process(target=model.simulate, args=(10, 50))

Providing target=model.simulate will pass a reference to the function itself rather than executing it and passing the result. This way it will be executed on the new process and you should acheive the parallelism.
See offical docs for an example.


Instruction: Python - PyTorch: IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices

Input:
I am working with PyTorch on a Text Classification problem with BERT. This is the PyTorch Dataset format I am using but when I try to access the inputs from the Dataset I get an error.
PyTorch Dataset
The Dataset Returns a Dictionary containing : ids, mask, token_type_ids, targets
class JigsawDataset:
    def __init__(self, df, train_transforms = None):
        self.comment_text = df["comment_text"].values
        self.target = df["toxic"].values
        self.tokenizer = config.BERT_TOKENIZER
        self.max_len = config.MAX_LEN
        self.langs = df["lang"].values
        self.train_transforms = train_transforms

    def __len__(self):
        return len(self.comment_text)

    def __getitem__(self, item):
        comment_text = str(self.comment_text[item])
        comment_text = " ".join(comment_text.split())
        lang = self.langs[item]

        if self.train_transforms:
            comment_text, _ = self.train_transforms(data=(comment_text, lang))['data']

        inputs = self.tokenizer.encode_plus(
            comment_text,
            None,
            add_special_tokens=True,
            max_length=self.max_len,
            pad_to_max_length=True,
            truncation=True,
        )

        ids = inputs["input_ids"]
        mask = inputs["attention_mask"]
        token_type_ids = inputs["token_type_ids"]

        data_loader_dict = {}
        data_loader_dict["ids"] = torch.tensor(ids, dtype=torch.long)
        data_loader_dict["mask"] = torch.tensor(mask, dtype=torch.long)
        data_loader_dict["token_type_ids"] = torch.tensor(token_type_ids, dtype=torch.long)
        data_loader_dict["targets"] = torch.tensor(self.target[item], dtype=torch.float)

        return data_loader_dict

Relevant Code which Gives Error
In this case I am trying to load only 1 Sample and make it to the format of the PyTorch Dataset
df = pd.read_csv("dataset.csv")
df = df.head(1) # Trying with only 1 Sample
dataset = JigsawDataset(df)

ids = dataset["ids"]    # Error occurs at this line
mask = dataset["mask"]
token_type_ids = ["token_type_ids"]

Error
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-78-4608dd623cac> in <module>
      3 dataset = JigsawDataset(df)
      4
----> 5 ids = dataset["ids"]    # Error occurs at this line
      6 mask = dataset["mask"]
      7 token_type_ids = ["token_type_ids"]

<ipython-input-40-121d8aa71516> in __getitem__(self, item)
     13
     14     def __getitem__(self, item):
---> 15         comment_text = str(self.comment_text[item])
     16         comment_text = " ".join(comment_text.split())
     17         lang = self.langs[item]

IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices

How to fix this?

Output:
I figured out the problem.
Incorrect Code
ids = dataset["ids"]
mask = dataset["mask"]
token_type_ids = ["token_type_ids"]

Correct Code
ids = dataset[0]["ids"]
mask = dataset[0]["mask"]
token_type_ids = [0]["token_type_ids"]

The problem was that "ids", "mask" and "token_type_ids" are Dictionary Keys. JigsawDataset returns a dictionary for each sample. So in order to access a sample we need to specify the index ([0]) before specifying the key.


Instruction: Pytorch issue with loss and number of epochs

Input:
I'm building a neural network by adapting the code shown in curiosily's tutorial. Instead of using weather data, I'm feeding in my own data (all numerical) to solve a time-series regression problem. Under the Finding Good Parameters section, they calculate the loss (difference between calculated and actual output values).
With my data (and using different optimizer, no. nodes, no. layers, etc.), the Train set - loss and Test  set - loss values can decrease with the no. epochs, then the loss values increase again. The accuracy is always 0.0. I want to understand why this happens, what would be an ideal loss value (zero?), and how I can adjust my model parameters to avoid this issue.
I'm basically using the same code in the tutorial, with a different neural network:
class Net(nn.Module):

    def __init__(self, n_features):
        super(Net, self).__init__()
        # n_features = no. inputs
        n1 = 8 # no. nodes in layer 1
        n2 = 5
        n3 = 4
        n4 = 5
        n5 = 2
        self.fc1 = nn.Linear(n_features,n1)
        self.fc2 = nn.Linear(n1,n2)
        self.fc3 = nn.Linear(n2,n3)
        self.fc4 = nn.Linear(n3,n4)
        self.fc5 = nn.Linear(n4,n5)
        self.fc6 = nn.Linear(n5,1)

    def forward(self, x):
        #x = F.relu(self.fc1(x))
        x = torch.tanh(self.fc1(x)) # activation function in layer 1
        x = torch.sigmoid(self.fc2(x))
        x = torch.sigmoid(self.fc3(x))
        x = torch.sigmoid(self.fc4(x))
        x = torch.tanh(self.fc5(x))
        return torch.sigmoid(self.fc6(x))

For the training/testing data,
print(X_train.shape, y_train.shape)
print(X_test.shape, y_test.shape)

gives
torch.Size([20, 8]) torch.Size([20])
torch.Size([6, 8]) torch.Size([6])

Here's some of my data:
                 Price        f1          f2         f3           f4  \
Date
2015-03-02   90.196107  1803.892  113.146970  12.643646  2125.656231
2015-03-09   64.135647  1800.734  107.968714   5.875968  2121.790735
2015-03-16   79.552756  1704.983  110.304459  12.003638  2009.193045
2015-03-23   82.191813  1607.716  107.720195   6.442494  2020.463010
2015-03-30   69.386627  1522.380  108.315439  13.252422  1979.088367
2016-03-07   66.651752  2084.698  113.987594  15.707330  2101.044023
2016-03-14   65.263433  2089.886  110.828986  10.185968  2126.727206
2016-03-21   67.420919  2152.666  111.177730   8.500986  2167.854746
2016-03-28   41.540860  2280.450   95.394193  11.750658  2103.708359
2017-03-06   45.244413  2383.778  110.464190  21.425014  2053.123167
2017-03-13   54.460675  2289.858  109.539569  10.345976  1982.583561
2017-03-20   41.063493  2185.491  106.347338  25.485176  1946.495832
2017-03-27   49.431981  2087.931  110.003395  10.732664  2032.264678
2018-03-05   73.660636  2204.947  108.703186   5.965236  2017.757273
2018-03-12   65.089474  2244.313  105.978320  11.164498  2102.231834
2018-03-19   61.284307  2240.600  106.864093   8.307786  2130.436459
2018-03-26   57.872814  2256.034  107.546072  16.750366  2153.384082
2019-03-04  173.318212  1826.327  113.837832  16.328690  2130.480772
2019-03-11  199.718808  1789.397  110.402293   6.385144  2038.025531
2019-03-18  206.258064  1809.019  109.644544   4.469384  1957.963904
2019-03-25  186.447336  1779.967  111.211074  17.378698  1948.683384
2020-03-02   63.820617  2586.044  113.275140   8.278228  2108.441593
2020-03-09   52.762931  2513.891  111.669942  12.933696  2087.767817
2020-03-16   72.150978  2467.322  109.775070  15.961352  2058.925025
2020-03-23   75.902965  2394.069  111.015771  18.886624  2023.038540
2020-03-30   51.715278  2298.855   95.129930  10.840378  2122.552675

                    f5          f6  year  week
Date
2015-03-02  321349.480  232757.674  2015    10
2015-03-09  319000.479  221875.266  2015    11
2015-03-16  329682.915  226521.004  2015    12
2015-03-23  323335.102  221358.104  2015    13
2015-03-30  335423.556  222942.088  2015    14
2016-03-07  324917.837  235534.038  2016    10
2016-03-14  318739.973  229351.230  2016    11
2016-03-21  311516.881  231233.470  2016    12
2016-03-28  317998.580  198436.598  2016    13
2017-03-06  333304.312  227996.148  2017    10
2017-03-13  319538.063  225794.464  2017    11
2017-03-20  343361.214  219506.514  2017    12
2017-03-27  326703.683  227488.980  2017    13
2018-03-05  306569.458  225853.320  2018    10
2018-03-12  309483.605  219876.156  2018    11
2018-03-19  316931.421  221450.730  2018    12
2018-03-26  322248.386  224380.222  2018    13
2019-03-04  340449.937  235389.124  2019    10
2019-03-11  323107.510  227822.394  2019    11
2019-03-18  322681.705  226564.046  2019    12
2019-03-25  342102.164  229219.588  2019    13
2020-03-02  343116.127  234588.908  2020    10
2020-03-09  345827.356  230804.352  2020    11
2020-03-16  341559.653  226640.770  2020    12
2020-03-23  344563.904  229330.532  2020    13
2020-03-30  327042.742  196731.040  2020    14

I split the data into training/testing sets:
# inputs
cols0 = [i for i in cols if i != 'Price']
X = mydata[cols0]

# output
y = mydata[['Price']]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=RANDOM_SEED)

Output:
The original post is working with a binary classification problem, where the accuracy metric makes sense (note that the predicted floats are first converted to a boolean tensor: predicted = y_pred.ge(.5).view(-1)).
On the other hand, your question stated that you are working with a regression problem, in which case accuracy doesn't really make sense. It's almost impossible to predict a float value exactly.


Instruction: TypeError: Int' object is not callable when calling Bert methods for producing embeddings

Input:
I have the following code and I obtain 'TypeError: 'tuple' object is not callable'(in new_time) but I dont understand why. I wrote it based on this tutorial
https://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/  and
https://github.com/getalp/Flaubert
My code :
#torch == 1.8.1
#numpy == 1.20.2
#pandas == 1.0.3
#transformers == 4.6.1

from transformers import logging
logging.set_verbosity_warning()

import numpy as np
import torch
from transformers import FlaubertModel, FlaubertTokenizer

language_model_dir = 'flaubert/flaubert_small_cased' # version > 2.0.0
flaubert, info = FlaubertModel.from_pretrained(language_model_dir, output_loading_info=True)
flaubert_tokenizer = FlaubertTokenizer.from_pretrained(language_model_dir)

# f_verbatim is a " <class 'pandas.core.series.Series'>", table of sentences
tokenized = f_verbatim.apply((lambda x: flaubert_tokenizer.encode(x, add_special_tokens=True, max_length=512, padding=True, truncation=True)))
#print(tokenized)

#Padding
max_len = 0
for i in tokenized.values:
    if len(i) > max_len:
        max_len = len(i)
padded = np.array([i + [0] * (max_len - len(i)) for i in tokenized.values])
# set data to tensor format
input_ids = torch.tensor(padded)
print(type(input_ids)) #<class 'torch.Tensor'>

attention_mask = np.where(padded != 0, 1, 0)
print(type(attention_mask))  #<class 'numpy.ndarray'>

# this line is causing the error
hidden_state = flaubert(input_ids, attention_mask=attention_mask)

Error :  #Stacktrace

--------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-7-8d53b819c31a> in <module>
      1 print(flaubert)
----> 2 hidden_state = flaubert(input_ids, attention_mask=attention_mask)

~\Anaconda3\envs\bert\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~\Anaconda3\envs\bert\lib\site-packages\transformers\models\flaubert\modeling_flaubert.py in forward(self, input_ids, attention_mask, langs, token_type_ids, position_ids, lengths, cache, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)
    195
    196         # generate masks
--> 197         mask, attn_mask = get_masks(slen, lengths, self.causal, padding_mask=attention_mask)
    198         # if self.is_decoder and src_enc is not None:
    199         #     src_mask = torch.arange(src_len.max(), dtype=torch.long, device=lengths.device) < src_len[:, None]

~\Anaconda3\envs\bert\lib\site-packages\transformers\models\xlm\modeling_xlm.py in get_masks(slen, lengths, causal, padding_mask)
    104
    105     # sanity check
--> 106     assert mask.size() == (bs, slen)
    107     assert causal is False or attn_mask.size() == (bs, slen, slen)
    108

TypeError: 'int' object is not callable

As I understand, the problem is due to a missing comma but I cannot figure it out.
printing "flaubert" fucntion give :
(FlaubertModel(
  (position_embeddings): Embedding(512, 512)
  (embeddings): Embedding(68729, 512, padding_idx=2)
  (layer_norm_emb): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
  (attentions): ModuleList(
    (0): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
    (1): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
    (2): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
    (3): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
    (4): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
    (5): MultiHeadAttention(
      (q_lin): Linear(in_features=512, out_features=512, bias=True)
      (k_lin): Linear(in_features=512, out_features=512, bias=True)
      (v_lin): Linear(in_features=512, out_features=512, bias=True)
      (out_lin): Linear(in_features=512, out_features=512, bias=True)
    )
  )
  (layer_norm1): ModuleList(
    (0): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (3): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (4): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (5): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
  )
  (ffns): ModuleList(
    (0): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
    (1): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
    (2): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
    (3): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
    (4): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
    (5): TransformerFFN(
      (lin1): Linear(in_features=512, out_features=2048, bias=True)
      (lin2): Linear(in_features=2048, out_features=512, bias=True)
    )
  )
  (layer_norm2): ModuleList(
    (0): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (3): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (4): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
    (5): LayerNorm((512,), eps=1e-06, elementwise_affine=True)
  )
), {'missing_keys': [], 'unexpected_keys': ['pred_layer.proj.bias', 'pred_layer.proj.weight'], 'error_msgs': []})

f_verbatim look likes this :
<class 'pandas.core.series.Series'>
0      Dans le cadre de l’ATEX, il y a certains types de départ moteur qu’on va mesurer la température de pot du moteur et en cas d’anomalie il faut absolument couper le moteur.
1      moi ce qui me dérange. C’est quand on a des enfants en bas âge. C’est dangereux, c’est trop facile
2      par rapport à une… enfin, à ce qui existe actuellement, si on parle du Tesys U… Enfin, sur Canopen, par exemple.
3      Je ne verrais pas ça pour une machine, on va dire, une application. Ce serait pour plusieurs machines.
4      Spécifique : Pas n’importe qui pourrait le prendre

attention_mask look like this :
tensor([[  0, 156,  20,  ...,   0,   0,   0],
        [  0, 253,  45,  ...,   0,   0,   0],
        [  0,  38, 243,  ...,   0,   0,   0],
        ...,
        [  0, 141, 104,  ...,   0,   0,   0],
        [  0,  59, 178,  ...,   0,   0,   0],
        [  0, 141, 432,  ...,   0,   0,   0]], dtype=torch.int32)

input_ids like this :
[[0 1 1 ... 0 0 0]
 [0 1 1 ... 0 0 0]
 [0 1 1 ... 0 0 0]
 ...
 [0 1 1 ... 0 0 0]
 [0 1 1 ... 0 0 0]
 [0 1 1 ... 0 0 0]]

Output:
This is because from_pretrained function gives you a tuple of model and dictionary and you did not separate them. Modify you code like this (add another variable):
flaubert, info = FlaubertModel.from_pretrained(language_model_dir, output_loading_info=True)

You have set output_loading_info to True. So it also return a dictionary. If you don't specify assignment variable, it will pass a tuple (model,dictionary) to flaubert variable. And since falubert is a tuple, you can not execute it.
UPDATE: attention_mask is a numpy array, but your model expects a torch tensor. So, convert it to a torch tensor before passing it to your model.
attention_mask = torch.from_numpy(attention_mask)
hidden_state = flaubert(input_ids, attention_mask=attention_mask)


Instruction: RuntimeError: Expected 4-dimensional input for 4-dimensional weight

Input:
I have a network, in which there are 3 architectures that share the same classifier.
class VGGBlock(nn.Module):
    def __init__(self, in_channels, out_channels,batch_norm=False):

        super(VGGBlock,self).__init__()

        conv2_params = {'kernel_size': (3, 3),
                        'stride'     : (1, 1),
                        'padding'   : 1
                        }

        noop = lambda x : x

        self._batch_norm = batch_norm

        self.conv1 = nn.Conv2d(in_channels=in_channels,out_channels=out_channels , **conv2_params)
        self.bn1 = nn.BatchNorm2d(out_channels) if batch_norm else noop

        self.conv2 = nn.Conv2d(in_channels=out_channels,out_channels=out_channels, **conv2_params)
        self.bn2 = nn.BatchNorm2d(out_channels) if batch_norm else noop

        self.max_pooling = nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))

    @property
    def batch_norm(self):
        return self._batch_norm

    def forward(self,x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = F.relu(x)

        x = self.conv2(x)
        x = self.bn2(x)
        x = F.relu(x)

        x = self.max_pooling(x)

        return x

class VGG16(nn.Module):

  def __init__(self, input_size, num_classes=1,batch_norm=False):
    super(VGG16, self).__init__()

    self.in_channels,self.in_width,self.in_height = input_size

    self.block_1 = VGGBlock(self.in_channels,64,batch_norm=batch_norm)
    self.block_2 = VGGBlock(64, 128,batch_norm=batch_norm)
    self.block_3 = VGGBlock(128, 256,batch_norm=batch_norm)
    self.block_4 = VGGBlock(256,512,batch_norm=batch_norm)

  @property
  def input_size(self):
      return self.in_channels,self.in_width,self.in_height

  def forward(self, x):

    x = self.block_1(x)
    x = self.block_2(x)
    x = self.block_3(x)
    x = self.block_4(x)

    return x

class VGG16Classifier(nn.Module):

  def __init__(self, num_classes=1,classifier = None,batch_norm=False):
    super(VGG16Classifier, self).__init__()

    self._vgg_a = VGG16((1,32,32),batch_norm=True)
    self._vgg_b = VGG16((1,32,32),batch_norm=True)
    self._vgg_star = VGG16((1,32,32),batch_norm=True)
    self.classifier = classifier

    if (self.classifier is None):
        self.classifier = nn.Sequential(
          nn.Linear(2048, 2048),
          nn.ReLU(True),
          nn.Dropout(p=0.5),
          nn.Linear(2048, 512),
          nn.ReLU(True),
          nn.Dropout(p=0.5),
          nn.Linear(512, num_classes)
        )

  def forward(self, x1,x2,x3):
      op1 = self._vgg_a(x1)
      op1 = torch.flatten(op1,1)
      op2 = self._vgg_b(x2)
      op2 = torch.flatten(op2,1)
      op3 = self._vgg_star(x3)
      op3 = torch.flatten(op3,1)

      x1 = self.classifier(op1)
      x2 = self.classifier(op2)
      x3 = self.classifier(op3)

      return x1,x2,x3

model1 = VGG16((1,32,32),batch_norm=True)
model2 = VGG16((1,32,32),batch_norm=True)
model_star = VGG16((1,32,32),batch_norm=True)
model_combo = VGG16Classifier(model1,model2,model_star)

I want to traing model_combo using the following loss function:
class CombinedLoss(nn.Module):
    def __init__(self, loss_a, loss_b, loss_star, _lambda=1.0):
        super().__init__()
        self.loss_a = loss_a
        self.loss_b = loss_b
        self.loss_star = loss_star

        self.register_buffer('_lambda',torch.tensor(float(_lambda),dtype=torch.float32))

    def forward(self,y_hat,y):

        return (self.loss_a(y_hat[0],y[0]) +
                self.loss_b(y_hat[1],y[1]) +
                self.loss_combo(y_hat[2],y[2]) +
                self._lambda * torch.sum(model_star.weight - torch.pow(torch.cdist(model1.weight+model2.weight), 2)))

In the training function I pass loaders, that for simplicity are loaders_a, loaders_b and again loaders_a, where loaders_a is related to the first 50% of data of MNIST and loaders_b to the latter 50% of MNIST.
def train(net, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
      loaders_a, loaders_b, loaders_star = loaders
    # try:
      net = net.to(dev)
      #print(net)
      #summary(net,[(net.in_channels,net.in_width,net.in_height)]*2)

      criterion.to(dev)

      # Initialize history
      history_loss = {"train": [], "val": [], "test": []}
      history_accuracy_a = {"train": [], "val": [], "test": []}
      history_accuracy_b = {"train": [], "val": [], "test": []}
      history_accuracy_star = {"train": [], "val": [], "test": []}
      # Store the best val accuracy
      best_val_accuracy = 0

      # Process each epoch
      for epoch in range(epochs):
        # Initialize epoch variables
        sum_loss = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_a = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_b = {"train": 0, "val": 0, "test": 0}
        sum_accuracy_star = {"train": 0, "val": 0, "test": 0}

        progbar = None
        # Process each split
        for split in ["train", "val", "test"]:
          if split == "train":
            net.train()
            #widgets = [
              #' [', pb.Timer(), '] ',
              #pb.Bar(),
              #' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')]

            #progbar = pb.ProgressBar(max_value=len(loaders_a[split]),widgets=widgets,redirect_stdout=True)

          else:
            net.eval()
          # Process each batch
          for j, ((input_a, labels_a), (input_b, labels_b), (input_s, labels_s)) in enumerate(zip(loaders_a[split], loaders_b[split], loaders_star[split])):
            labels_a = labels_a.unsqueeze(1).float()
            labels_b = labels_b.unsqueeze(1).float()
            labels_s = labels_s.unsqueeze(1).float()

            input_a = input_a.to(dev)
            labels_a = labels_a.to(dev)
            input_b = input_b.to(dev)
            labels_b = labels_b.to(dev)
            input_s = input_s.to(dev)
            labels_s = labels_s.to(dev)

            # Reset gradients
            optimizer.zero_grad()
            # Compute output
            pred = net(input_a,input_b, input_s)

            loss = criterion(pred, [labels_a, labels_b, labels_s])
            # Update loss
            sum_loss[split] += loss.item()
            # Check parameter update
            if split == "train":
              # Compute gradients
              loss.backward()
              # Optimize
              optimizer.step()

            # Compute accuracy
            pred_labels = (pred[2] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_a = (pred[0] >= 0.0).long()  # Binarize predictions to 0 and 1
            pred_labels_b = (pred[1] >= 0.0).long()  # Binarize predictions to 0 and 1

            batch_accuracy_star = (pred_labels == labels_s).sum().item() / len(labels_s)
            batch_accuracy_a = (pred_labels_a == labels_a).sum().item() / len(labels_a)
            batch_accuracy_b = (pred_labels_b == labels_b).sum().item() / len(labels_b)
            # Update accuracy
            sum_accuracy_star[split] += batch_accuracy_star
            sum_accuracy_a[split] += batch_accuracy_a
            sum_accuracy_b[split] += batch_accuracy_b

            #if (split=='train'):
              #progbar.update(j, ta=batch_accuracy)
              #progbar.update(j, ta=batch_accuracy_a)
              #progbar.update(j, ta=batch_accuracy_b)

        #if (progbar is not None):
          #progbar.finish()
        # Compute epoch loss/accuracy
        #for split in ["train", "val", "test"]:
          #epoch_loss = sum_loss[split] / (len(loaders_a[split])+len(loaders_b[split]))
          #epoch_accuracy_combo = {split: sum_accuracy_combo[split] / len(loaders[split]) for split in ["train", "val", "test"]}
          #epoch_accuracy_a = sum_accuracy_a[split] / len(loaders_a[split])
          #epoch_accuracy_b = sum_accuracy_b[split] / len(loaders_b[split])
        epoch_loss = sum_loss["train"] / (len(loaders_a["train"])+len(loaders_b["train"])+len(loaders_s["train"]))
        epoch_accuracy_a = sum_accuracy_a["train"] / len(loaders_a["train"])
        epoch_accuracy_b = sum_accuracy_b["train"] / len(loaders_b["train"])
        epoch_accuracy_star = sum_accuracy_star["train"] / len(loaders_s["train"])

        epoch_loss_val = sum_loss["val"] / (len(loaders_a["val"])+len(loaders_b["val"])+len(loaders_s["val"]))
        epoch_accuracy_a_val = sum_accuracy_a["val"] / len(loaders_a["val"])
        epoch_accuracy_b_val = sum_accuracy_b["val"] / len(loaders_b["val"])
        epoch_accuracy_star_val = sum_accuracy_star["val"] / len(loaders_s["val"])

        epoch_loss_test = sum_loss["test"] / (len(loaders_a["test"])+len(loaders_b["test"])+len(loaders_s["test"]))
        epoch_accuracy_a_test = sum_accuracy_a["test"] / len(loaders_a["test"])
        epoch_accuracy_b_test = sum_accuracy_b["test"] / len(loaders_b["test"])
        epoch_accuracy_star_test = sum_accuracy_star["test"] / len(loaders_s["test"])

        # Store params at the best validation accuracy
        if save_param and epoch_accuracy["val"] > best_val_accuracy:
          # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
          torch.save(net.state_dict(), f"{model_name}_best_val.pth")
          best_val_accuracy = epoch_accuracy["val"]

        # Update history
        for split in ["train", "val", "test"]:
          history_loss[split].append(epoch_loss)
          history_accuracy_a[split].append(epoch_accuracy_a)
          history_accuracy_b[split].append(epoch_accuracy_b)
          history_accuracy_star[split].append(epoch_accuracy_star)
        # Print info
        print(f"Epoch {epoch + 1}:",
              f"Training Loss = {epoch_loss:.4f},",)
        print(f"Epoch {epoch + 1}:",
              f"Training Accuracy for A = {epoch_accuracy_a:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Training Accuracy for B = {epoch_accuracy_b:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Training Accuracy for star = {epoch_accuracy_star:.4f},")

        print(f"Epoch {epoch + 1}:",
              f"Val Loss = {epoch_loss_val:.4f},",)
        print(f"Epoch {epoch + 1}:",
              f"Val Accuracy for A = {epoch_accuracy_a_val:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Val Accuracy for B = {epoch_accuracy_b_val:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Val Accuracy for star = {epoch_accuracy_star_val:.4f},")

        print(f"Epoch {epoch + 1}:",
              f"Test Loss = {epoch_loss_test:.4f},",)
        print(f"Epoch {epoch + 1}:",
              f"Test Accuracy for A = {epoch_accuracy_a_test:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Test Accuracy for B = {epoch_accuracy_b_test:.4f},")
        print(f"Epoch {epoch + 1}:",
              f"Test Accuracy for star = {epoch_accuracy_star_test:.4f},")
        print("\n")

But I got this error:
RuntimeError: Expected 4-dimensional input for 4-dimensional weight [64, 1, 3, 3], but got 2-dimensional input of size [128, 2048] instead

Output:
From your code & error, I guess you're passing binary image (h, w, 1) to the network.
The issue raises in Conv2d layer, where it expects 4 dimensional input.
To rephrase - Conv2d layer expects 4-dim tensor like:
T = torch.randn(1,3,128,256)
print(T.shape)
out: torch.Size([1, 3, 128, 256])

Where:

The first dimension (number 1) is batch dimension to stack multiple tensors across this dim to perform batch operation.
Second dimension (number 3) is in_channels for Conv. It's basically number of channels of image. Standard RGB or BGR image has 3 channels.
Third dimension (number 128) is height dimension.
Fourth dimension (number 256) is width dimension.

Binary images have 1 channel dimension:
[128, 256, 1] , [Height, Width, Channels]
OR
[128, 256], [Height, Width]

Take in consideration:
That standard Numpy image array dims have [H, W, C] shape, where torch expects Channel dimension after batch dimension, so: [B, C, H, W].
I'm not sure where channel clamping is happening, but binary image becomes 2-dim image, because there's no need for channel dimension as long as it's just one.
If you want to pass 2-dim binary image to Conv2d layer, you should unsqueeze it to 4-dim tensor.
Before: Input.shape = torch.Size[128, 2048]

Preprocess:
Tensor = Input.view(1, 1, Input.shape[0], Input.shape[1])
Tensor.shape
out: = torch.Size[1, 1, 128, 2048]

Same could be done by just unsqueezing zeroth dim two times:
Tensor = Input.unsqueeze(0).unsqueeze(0)
Tensor.shape
out: = torch.Size[1, 1, 128, 2048]

But it's messier - so I'd recommend first option.


Instruction: Both validation loss and accuracy are increasing using a pre-trained VGG-16

Input:
So, I'm doing a 4 label x-ray images classification on around 12600 images:
Class1:4000
Class2:3616
Class3:1345
Class4:4000
I'm using VGG-16 architecture pertained on the imageNet dataset with cross-entrpy and SGD and a batch size of 32 and a learning rate of 1e-3 running on pytorch

   [[749.,   6.,  50.,   2.],
    [  5., 707.,   9.,   1.],
    [ 56.,   8., 752.,   0.],
    [  4.,   1.,   0., 243.]]

I know since both train loss/acc are relatively 0/1 the model is overfitting, though I'm surprised that the val acc is still around 0.9!
How to properly interpret that and what causing it and how to prevent it?
I know it's something like because the accuracy is the argmax of softmax like the actual predictions are getting lower and lower but the argmax always stays the same, but I'm really confused about it! I even let it train for +64 epochs same results flat acc while loss increases gradually!
PS. I have seen other questions with answers and didn't really get an explanation

Output:
I think your question already says about what is going on. Your model is overfitting as you have also figured out. Now, as you are training more your model slowly becoming more specialized to the train set and loosing the the capability to generalize gradually. So the softmax probabilities are getting more and more flat. But still it is showing more or less the same accuracy for validation set as still now the correct class has at least slightly more probability than the others. So in my opinion there can be some possible reasons for this:

Your train set and validation set may not be from the same distribution.
Your validation set doesn't cover all cases need to be evaluated, it probably contains similar types of images but they do not differ too much. So, when the model can identify one, it can identify many of them from the validation set. If you add more heterogeneous images in validation set, you will no longer see such a large accuracy in validation set.
Similarly, we can say your train set has images which are heterogeneous i.e, they have a lot of variations, and the validation set is covering only a few varieties, so as training goes on, those minorities are getting less priority as the model yet to have many things to learn and generalize. This can happen if you augment your train-set and your model finds the validation set is relatively easier initially (until overfitting), but as training goes on the model gets lost itself while learning a lot of augmented varieties available in the train set. In this case don't make the augmentation too much wild. Think, if the augmented images are still realistic or not. Do augmentation on images as long as they remain realistic and each type of these images' variations occupy enough representative examples in the train set. Don't include unnecessary situations in augmentation those will never occur in reality, as these unrealistic examples will just increase burden on the model than doing any help.


Instruction: Unable to install Pytorch on Mac OS X from scratch due to Pytorch package conflicts with Conda - how to fix?

Input:
I have python 3.9 and I am trying to install pytorch current version (as of this writing 1.9). But when I do it I get the following error:
(synthesis) miranda9@Brandos-MBP ~ % conda install pytorch torchvision torchaudio -c pytorch
Collecting package metadata (current_repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.
Solving environment: failed with repodata from current_repodata.json, will retry with next repodata source.
Collecting package metadata (repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.
Solving environment: -
Found conflicts! Looking for incompatible packages.                                                                                failed

UnsatisfiableError: The following specifications were found to be incompatible with each other:

Output in format: Requested package -> Available versions

Package pytorch conflicts for:
torchaudio -> pytorch[version='1.2.0|1.3.0|1.3.1|1.4.0|1.5.0|1.5.1|1.6.0|1.7.0|1.7.1|1.8.0|1.8.1|1.9.0']
torchvision -> pytorch[version='1.2.0|1.3.0|1.3.1|1.4.0|1.5.0|1.5.1|1.6.0|1.7.0|1.7.1|1.8.0|1.8.1|1.9.0|>=1.1.0|>=1.0.0|>=0.4|>=0.3|>=0.2|1.7.1.*|1.3.1.*']
pytorch

Package six conflicts for:
torchvision -> six
pytorch -> mkl-service[version='>=2,<3.0a0'] -> six

I only had numpy installed so far...it's essentially a brand new env:
(synthesis) miranda9@Brandos-MBP ~ % conda list
# packages in environment at /Users/miranda9/.conda/envs/synthesis:
#
# Name                    Version                   Build  Channel
blas                      1.0                         mkl
ca-certificates           2021.5.25            hecd8cb5_1
certifi                   2021.5.30        py39hecd8cb5_0
intel-openmp              2021.2.0           hecd8cb5_564
libcxx                    10.0.0                        1
libffi                    3.3                  hb1e8313_2
mkl                       2021.2.0           hecd8cb5_269
mkl-service               2.3.0            py39h9ed2024_1
mkl_fft                   1.3.0            py39h4a7008c_2
mkl_random                1.2.1            py39hb2f4e1b_2
ncurses                   6.2                  h0a44026_1
numpy                     1.20.2           py39h4b4dc7a_0
numpy-base                1.20.2           py39he0bd621_0
openssl                   1.1.1k               h9ed2024_0
pip                       21.1.2           py39hecd8cb5_0
python                    3.9.5                h88f2d9e_3
readline                  8.1                  h9ed2024_0
setuptools                52.0.0           py39hecd8cb5_0
six                       1.16.0             pyhd3eb1b0_0
sqlite                    3.35.4               hce871da_0
tk                        8.6.10               hb0a8c7a_0
tzdata                    2020f                h52ac0ba_0
wheel                     0.36.2             pyhd3eb1b0_0
xz                        5.2.5                h1de35cc_0
zlib                      1.2.11               h1de35cc_3

why is this happening and how do I fix this?

related/crossposted:

https://www.reddit.com/r/pytorch/comments/o1hwgv/installing_pytorch_fails_on_macos_with_brand_new/
SO: python - Unable to install Pytorch on Mac OS X from scratch due to Pytorch package conflicts with Conda - how to fix? - Stack Overflow
pytorch forum: https://discuss.pytorch.org/t/installing-pytorch-fails-on-macos/109361/3
UnsatisfiableError: The following specifications were found to be incompatible with each other:

Output:
For me it seems that adding conda-forge to the channels works. My understanding of why that works is that the pytorch channel doesn't have all packages or something (details here: https://github.com/pytorch/pytorch/issues/59517).
Do:
conda install -y pytorch torchvision torchaudio -c pytorch -c conda-forge

other example installations:
conda install -y pytorch==1.7.1 torchvision torchaudio cudatoolkit=10.2 -c pytorch -c conda-forge

Full output:
(synthesis) miranda9@Brandos-MBP ~ % conda install pytorch torchvision torchaudio -c pytorch -c conda-forge
Collecting package metadata (current_repodata.json): done
Solving environment: done

## Package Plan ##

  environment location: /Users/miranda9/.conda/envs/synthesis

  added / updated specs:
    - pytorch
    - torchaudio
    - torchvision

The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    bzip2-1.0.8                |       h0d85af4_4         155 KB  conda-forge
    ca-certificates-2021.5.30  |       h033912b_0         136 KB  conda-forge
    certifi-2021.5.30          |   py39h6e9494a_0         141 KB  conda-forge
    ffmpeg-4.3                 |       h0a44026_0        10.1 MB  pytorch
    freetype-2.10.4            |       h4cff582_1         890 KB  conda-forge
    gettext-0.19.8.1           |    h7937167_1005         3.3 MB  conda-forge
    gmp-6.1.2                  |    h0a44026_1000         734 KB  conda-forge
    gnutls-3.6.13              |       hc269f14_0         2.1 MB  conda-forge
    lame-3.100                 |    h35c211d_1001         521 KB  conda-forge
    libiconv-1.16              |       haf1e3a3_0         1.3 MB  conda-forge
    libpng-1.6.37              |       h7cec526_2         313 KB  conda-forge
    libuv-1.41.0               |       hbcf498f_0         421 KB  conda-forge
    libwebp-base-1.2.0         |       h0d85af4_2         700 KB  conda-forge
    lz4-c-1.9.2                |       h4a8c4bd_1         169 KB  conda-forge
    nettle-3.4.1               |    h3efe00b_1002         1.0 MB  conda-forge
    ninja-1.10.2               |       hf7b0b51_1         106 KB
    olefile-0.46               |     pyh9f0ad1d_1          32 KB  conda-forge
    openh264-2.1.1             |       hd174df1_0         1.5 MB  conda-forge
    openssl-1.1.1k             |       h0d85af4_0         1.9 MB  conda-forge
    pillow-8.2.0               |   py39h5270095_0         587 KB
    python_abi-3.9             |           1_cp39           4 KB  conda-forge
    pytorch-1.9.0              |          py3.9_0        79.0 MB  pytorch
    torchaudio-0.9.0           |             py39         4.0 MB  pytorch
    torchvision-0.10.0         |         py39_cpu         6.8 MB  pytorch
    typing_extensions-3.10.0.0 |     pyha770c72_0          28 KB  conda-forge
    ------------------------------------------------------------
                                           Total:       115.8 MB

The following NEW packages will be INSTALLED:

  bzip2              conda-forge/osx-64::bzip2-1.0.8-h0d85af4_4
  ffmpeg             pytorch/osx-64::ffmpeg-4.3-h0a44026_0
  freetype           conda-forge/osx-64::freetype-2.10.4-h4cff582_1
  gettext            conda-forge/osx-64::gettext-0.19.8.1-h7937167_1005
  gmp                conda-forge/osx-64::gmp-6.1.2-h0a44026_1000
  gnutls             conda-forge/osx-64::gnutls-3.6.13-hc269f14_0
  jpeg               pkgs/main/osx-64::jpeg-9b-he5867d9_2
  lame               conda-forge/osx-64::lame-3.100-h35c211d_1001
  lcms2              pkgs/main/osx-64::lcms2-2.12-hf1fd2bf_0
  libiconv           conda-forge/osx-64::libiconv-1.16-haf1e3a3_0
  libpng             conda-forge/osx-64::libpng-1.6.37-h7cec526_2
  libtiff            pkgs/main/osx-64::libtiff-4.2.0-h87d7836_0
  libuv              conda-forge/osx-64::libuv-1.41.0-hbcf498f_0
  libwebp-base       conda-forge/osx-64::libwebp-base-1.2.0-h0d85af4_2
  lz4-c              conda-forge/osx-64::lz4-c-1.9.2-h4a8c4bd_1
  nettle             conda-forge/osx-64::nettle-3.4.1-h3efe00b_1002
  ninja              pkgs/main/osx-64::ninja-1.10.2-hf7b0b51_1
  olefile            conda-forge/noarch::olefile-0.46-pyh9f0ad1d_1
  openh264           conda-forge/osx-64::openh264-2.1.1-hd174df1_0
  pillow             pkgs/main/osx-64::pillow-8.2.0-py39h5270095_0
  python_abi         conda-forge/osx-64::python_abi-3.9-1_cp39
  pytorch            pytorch/osx-64::pytorch-1.9.0-py3.9_0
  torchaudio         pytorch/osx-64::torchaudio-0.9.0-py39
  torchvision        pytorch/osx-64::torchvision-0.10.0-py39_cpu
  typing_extensions  conda-forge/noarch::typing_extensions-3.10.0.0-pyha770c72_0
  zstd               pkgs/main/osx-64::zstd-1.4.5-h41d2c2f_0

The following packages will be UPDATED:

  ca-certificates    pkgs/main::ca-certificates-2021.5.25-~ --> conda-forge::ca-certificates-2021.5.30-h033912b_0

The following packages will be SUPERSEDED by a higher-priority channel:

  certifi            pkgs/main::certifi-2021.5.30-py39hecd~ --> conda-forge::certifi-2021.5.30-py39h6e9494a_0
  openssl              pkgs/main::openssl-1.1.1k-h9ed2024_0 --> conda-forge::openssl-1.1.1k-h0d85af4_0

Proceed ([y]/n)? y

Downloading and Extracting Packages
freetype-2.10.4      | 890 KB    | ############################################################################################################################################################################################################################################################################# | 100%
openh264-2.1.1       | 1.5 MB    | ############################################################################################################################################################################################################################################################################# | 100%
openssl-1.1.1k       | 1.9 MB    | ############################################################################################################################################################################################################################################################################# | 100%
gmp-6.1.2            | 734 KB    | ############################################################################################################################################################################################################################################################################# | 100%
gnutls-3.6.13        | 2.1 MB    | ############################################################################################################################################################################################################################################################################# | 100%
gettext-0.19.8.1     | 3.3 MB    | ############################################################################################################################################################################################################################################################################# | 100%
libuv-1.41.0         | 421 KB    | ############################################################################################################################################################################################################################################################################# | 100%
libpng-1.6.37        | 313 KB    | ############################################################################################################################################################################################################################################################################# | 100%
olefile-0.46         | 32 KB     | ############################################################################################################################################################################################################################################################################# | 100%
python_abi-3.9       | 4 KB      | ############################################################################################################################################################################################################################################################################# | 100%
certifi-2021.5.30    | 141 KB    | ############################################################################################################################################################################################################################################################################# | 100%
pillow-8.2.0         | 587 KB    | ############################################################################################################################################################################################################################################################################# | 100%
torchaudio-0.9.0     | 4.0 MB    | ############################################################################################################################################################################################################################################################################# | 100%
lz4-c-1.9.2          | 169 KB    | ############################################################################################################################################################################################################################################################################# | 100%
pytorch-1.9.0        | 79.0 MB   | ############################################################################################################################################################################################################################################################################# | 100%
typing_extensions-3. | 28 KB     | ############################################################################################################################################################################################################################################################################# | 100%
ffmpeg-4.3           | 10.1 MB   | ############################################################################################################################################################################################################################################################################# | 100%
lame-3.100           | 521 KB    | ############################################################################################################################################################################################################################################################################# | 100%
torchvision-0.10.0   | 6.8 MB    | ############################################################################################################################################################################################################################################################################# | 100%
libwebp-base-1.2.0   | 700 KB    | ############################################################################################################################################################################################################################################################################# | 100%
ca-certificates-2021 | 136 KB    | ############################################################################################################################################################################################################################################################################# | 100%
bzip2-1.0.8          | 155 KB    | ############################################################################################################################################################################################################################################################################# | 100%
nettle-3.4.1         | 1.0 MB    | ############################################################################################################################################################################################################################################################################# | 100%
ninja-1.10.2         | 106 KB    | ############################################################################################################################################################################################################################################################################# | 100%
libiconv-1.16        | 1.3 MB    | ############################################################################################################################################################################################################################################################################# | 100%
Preparing transaction: done
Verifying transaction: done
Executing transaction: done


Instruction: Why Resnet model in tensorflow and pytorch give different feature length?

Input:
I'm trying to extract features of images through Resnet models pretrained on imagenet dataset as for the network should give the length of 2048 features. When I experimented with TensorFlow it gave the same amount of feature-length but when I try PyTorch version Resnet it gives me the length of 1000.
codes are as below
for Tensorflow
import numpy as np
from numpy.linalg import norm
import pickle
from tqdm import tqdm, tqdm_notebook
import os
import random
import time
import math
import tensorflow
from tensorflow.keras.preprocessing import image
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.resnet50 import ResNet50, preprocess_input
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.mobilenet import MobileNet
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAveragePooling2D

def model_picker(name):
    if (name == 'vgg16'):
        model = VGG16(weights='imagenet',
                      include_top=False,
                      input_shape=(224, 224, 3),
                      pooling='max')
    elif (name == 'vgg19'):
        model = VGG19(weights='imagenet',
                      include_top=False,
                      input_shape=(224, 224, 3),
                      pooling='max')
    elif (name == 'mobilenet'):
        model = MobileNet(weights='imagenet',
                          include_top=False,
                          input_shape=(224, 224, 3),
                          pooling='max',
                          depth_multiplier=1,
                          alpha=1)
    elif (name == 'inception'):
        model = InceptionV3(weights='imagenet',
                            include_top=False,
                            input_shape=(224, 224, 3),
                            pooling='max')
    elif (name == 'resnet'):
        model = ResNet50(weights='imagenet',
                         include_top=False,
                         input_shape=(224, 224, 3),
                        pooling='max')
    elif (name == 'xception'):
        model = Xception(weights='imagenet',
                         include_top=False,
                         input_shape=(224, 224, 3),
                         pooling='max')
    else:
        print("Specified model not available")
    return model

model_architecture = 'resnet'
model = model_picker(model_architecture)

def extract_features(img_path, model):
    input_shape = (224, 224, 3)
    img = image.load_img(img_path,
                         target_size=(input_shape[0], input_shape[1]))
    img_array = image.img_to_array(img)
    expanded_img_array = np.expand_dims(img_array, axis=0)
    preprocessed_img = preprocess_input(expanded_img_array)
    features = model.predict(preprocessed_img)
    flattened_features = features.flatten()
    normalized_features = flattened_features / norm(flattened_features)
    return normalized_features
features = extract_features('dog.jpg', model)
print(len(features))

> 2048

As you can see it gives a length of 2048 features through the resnet50 model
Below is the code for PyTorch
from torchvision import models, transforms
from PIL import Image
from torch.autograd import Variable
import torch
res_model = models.resnet50(pretrained=True)
def image_loader(image,model,use_gpu= False):
  transform = transforms.Compose([
                                  transforms.Resize(256),
                                  transforms.CenterCrop(224),
                                  transforms.ToTensor()
  ])
  img = Image.open(image)
  img = transform(img)
  print(img.shape)

  x = Variable(torch.unsqueeze(img, dim = 0).float(), requires_grad = False)
  print(x.shape)
  if use_gpu:
    x = x.cuda()
    model = model.cuda()
  y = model(x).cpu()
  print(y.size())
  y = torch.squeeze(y)
  y = y.data.numpy()
  print(y.shape)
  print(len(y))
  np.savetxt('features.txt',y,delimiter=',')
image_loader('dog.jpg',res_model)

> torch.Size([3, 224, 224]) torch.Size([1, 3, 224, 224]) torch.Size([1,
> 1000]) (1000,) 1000

As you can see it gives a length of 1000 for the feature extracted through the Resnet model with the PyTorch model
why am I getting different lengths isn't I get the same length according to architecture which is 2048 or am I doing anything wrong?

Output:
Printing the layers of the pytorch resnet will yield:
(fc): Linear(in_features=2048, out_features=1000, bias=True)

as the last layer of the resnet in Pytorch, because the model is by default set up for use as a classifier on imagenet data (1000 classes). If you want 2048 features instead, you can simply delete this last layer.
del model.fc

and your resulting output will then be of the desired dimension.
Edit: perhaps better is to simply overwrite model.fc with an identity function rather than deleting it so it doesn't cause errors when forward is called:
model.fc = torch.nn.Identity()


Instruction: Error while creating train transform using torch vision

Input:
I am using torch vision to create the following train transformation. I do not understand what's wrong and how I can fix it?
train_transform = torch.nn.Sequential(
    transforms.ToTensor(),
    transforms.RandomApply([
         transforms.RandomApply([transforms.RandomRotation(15)], 0.6),
         transforms.RandomApply([transforms.RandomAffine(0, shear=0.2)], 0.3),
         transforms.RandomApply([transforms.RandomResizedCrop(size = 224, scale=(0.8, 1.4))], 0.7)
        ], p= 0.5)
)

This is the error:

TypeError                                 Traceback (most recent call
last)  in ()
5          transforms.RandomApply([transforms.RandomAffine(0, shear=0.2)], 0.3),
6          transforms.RandomApply([transforms.RandomResizedCrop(size = 224,
scale=(0.8, 1.4))], 0.7)
----> 7         ], p= 0.5)
8 )
1 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in
add_module(self, name, module)
371         if not isinstance(module, Module) and module is not None:
372             raise TypeError("{} is not a Module subclass".format(
--> 373                 torch.typename(module)))
374         elif not isinstance(name, torch._six.string_classes):
375             raise TypeError("module name should be a string. Got {}".format(
TypeError: torchvision.transforms.transforms.ToTensor is not a Module
subclass

Output:
torch.nn.Sequential scripts your transformations. You can only use scriptable transformations in torch.nn.Sequential and transforms.ToTensor() is not a scriptable transformation.
A scriptable transformation only takes a Tensor as an input. This is why you cannot use transforms.ToTensor() in the torch.nn.Sequential function, as you take an some data as input and transform it to a tensor.
You can find this information in the PyTorch documentation here under Scriptable Transformations.  The transforms.RandomApply transformation is also not scriptable but it says here in the transforms.visions docu how to use it in a scriptable way anyway (see the definition of the class RandomApply). In the source code it says: " In order to script the transformation, please use torch.nn.ModuleList as input instead of list/tuple of transforms"
You have to wrap all your random transformations with torch.nn.ModuleList.
my_transforms = transforms.RandomApply(torch.nn.ModuleList([
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomRotation(15)]), p=0.6),
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomAffine(0, shear=0.2)]), p=0.3),
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomResizedCrop(size = 224, scale=(0.8, 1.4))]), p=0.7),

 ]), p=0.3)
scripted_transforms = torch.jit.script(my_transforms)

If you want to, you can also wrap these transformations with torch.nn.Sequential but it is not necessary. Then the code should be this:
my_transforms = torch.nn.Sequential(transforms.RandomApply(torch.nn.ModuleList([
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomRotation(15)]), p=0.6),
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomAffine(0, shear=0.2)]), p=0.3),
     transforms.RandomApply( torch.nn.ModuleList([ transforms.RandomResizedCrop(size = 224, scale=(0.8, 1.4))]), p=0.7),

 ]), p=0.3))
scripted_transforms = torch.jit.script(my_transforms)

As mentioned, the tensor transformation cannot be added to this, since it is not scriptable.
Another solution is to use the transforms.Compose instead of torch.nn.Sequential like this:
from torchvision import transforms
train_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.RandomApply([
         transforms.RandomApply([transforms.RandomRotation(15)], 0.6),
         transforms.RandomApply([transforms.RandomAffine(0, shear=0.2)], 0.3),
         transforms.RandomApply([transforms.RandomResizedCrop(size = 224, scale=(0.8, 1.4))], 0.7)
        ], p= 0.5)
])

In both cases the complete list of transformations will be randomly applied to your data, exactly in the order as you have specified the transformations in the list. You can see this from the source code of Random.Apply, where the forward pass looks like this
def forward(self, img):
        if self.p < torch.rand(1):
            return img
        for t in self.transforms:
            img = t(img)
        return img

The function loops over all transformations t in self.transforms exactly in the order you have specified in the list. So technically it is not necessary that you use torch.nn.Sequential.


Instruction: Equivalent AdaptiveAvgPool2d API in cuDNN

Input: Is there an equivalent API in cuDNN as the AdaptiveAvgPool2d in Pytorch?

Output:
yes, it's possible. you can create the pooling descriptor.
here is the official documentation for the API-
https://docs.nvidia.com/deeplearning/cudnn/api/index.html#cudnnPoolingMode_t