Instruction: Getting (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

Input:
class ConvBlock(nn.Module):
    def __init__(self, in_channels, out_channels, down=True, use_act=True, **kwargs):
        super().__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, padding_mode="reflect", **kwargs)
            if down
            else nn.ConvTranspose2d(in_channels, out_channels, **kwargs),
            nn.InstanceNorm2d(out_channels),
            nn.ReLU(inplace=True) if use_act else nn.Identity()
        )

    def forward(self, x):
        return self.conv(x)

class ResidualBlock(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.block = nn.Sequential(
            ConvBlock(channels, channels, kernel_size=3, padding=1),
            ConvBlock(channels, channels, use_act=False, kernel_size=3, padding=1),
        )

    def forward(self, x):
        return x + self.block(x)

class Generator(nn.Module):
    def __init__(self, image_channels, num_features= 64, num_residuals=9):
        super().__init__()
        self.initial = nn.Sequential(
            nn.Conv2d(image_channels, num_features, kernel_size=7, stride=1, padding=3, padding_mode="reflect"),
            nn.ReLU(inplace=True)
        )
        self.down_blocks = nn.ModuleList = ([
            ConvBlock(num_features, num_features*2, kernel_size=3, stride=2, padding=1),
            ConvBlock(num_features*2, num_features*4, kernel_size=3, stride=2, padding=1),
        ])

        self.residual_blocks = nn.Sequential(
            *[ResidualBlock(num_features*4) for _ in range(num_residuals)]
        )

        self.up_blocks = nn.ModuleList = ([
            ConvBlock(num_features*4, num_features*2, down=False, kernel_size=3, padding=1, stride=2, output_padding=1),
            ConvBlock(num_features*2, num_features, down=False, kernel_size=3, padding=1, stride=2, output_padding=1),
        ])

        self.last = nn.Conv2d(num_features, image_channels, kernel_size=7, stride=1, padding=3, padding_mode="reflect")

    def forward(self, x):
        x = self.initial(x)

        for layer in self.down_blocks:
            x = layer(x)

        x = self.residual_blocks(x)

        for layer in self.up_blocks:
            x = layer(x)

        return torch.tanh(self.last(x))

img_channels = 3
img_size = 256
x = torch.randn((2, img_channels, img_size, img_size))
x = x.to(DEVICE)
gen = Generator(img_channels, 9).to(DEVICE)
print(gen(x).shape)

I have implemented this model for Cycle GAN. The input used here is just for demonstration purposes to shorten the code however the actual input throws the same error. The code runs fine on the CPU but when I shift it to GPU I get the following error:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-30-a56669856674> in <module>
      4 x = x.to(DEVICE)
      5 gen = Generator(img_channels, 9).to(DEVICE)
----> 6 print(gen(x).shape)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

<ipython-input-25-f9a41f6c9d12> in forward(self, x)
     26
     27         for layer in self.down_blocks:
---> 28             x = layer(x)
     29
     30         x = self.residual_blocks(x)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

<ipython-input-23-e139087b9df4> in forward(self, x)
     11
     12     def forward(self, x):
---> 13         return self.conv(x)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    115     def forward(self, input):
    116         for module in self:
--> 117             input = module(input)
    118         return input
    119

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    421
    422     def forward(self, input: Tensor) -> Tensor:
--> 423         return self._conv_forward(input, self.weight)
    424
    425 class Conv3d(_ConvNd):

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    416             return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
    417                             weight, self.bias, self.stride,
--> 418                             _pair(0), self.dilation, self.groups)
    419         return F.conv2d(input, weight, self.bias, self.stride,
    420                         self.padding, self.dilation, self.groups)

RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

My model is being changed to device(CUDA) and the input as well. I have no idea what the issue is now

Output:
You have typos in your code:
self.down_blocks = nn.ModuleList = ([
...
self.up_blocks = nn.ModuleList = ([

should be:
self.down_blocks = nn.ModuleList([
...
self.up_blocks = nn.ModuleList([

You need to reload your kernel since at this point you've essentially overwritten nn.ModuleList to a list.


Instruction: PyTorch's torch.as_strided with negative strides for making a Toeplitz matrix

Input:
I am writing a jury-rigged PyTorch version of scipy.linalg.toeplitz, which currently has the following form:
def toeplitz_torch(c, r=None):
    c = torch.tensor(c).ravel()
    if r is None:
        r = torch.conj(c)
    else:
        r = torch.tensor(r).ravel()

    # Flip c left to right.
    idx = [i for i in range(c.size(0)-1, -1, -1)]
    idx = torch.LongTensor(idx)
    c = c.index_select(0, idx)

    vals = torch.cat((c, r[1:]))
    out_shp = len(c), len(r)
    n = vals.stride(0)

    return torch.as_strided(vals[len(c)-1:], size=out_shp, stride=(-n, n)).copy()

But torch.as_strided currently does not support negative strides. My function, therefore, throws the error:

RuntimeError: as_strided: Negative strides are not supported at the moment, got strides: [-1, 1].

My (perhaps incorrect) understanding of as_strided is that it inserts the values of the first argument into a new array whose size is specified by the second argument and it does so by linearly indexing those values in the original array and placing them at subscript-indexed strides given by the final argument.
Both the NumPy and PyTorch documentation concerning as_strided have scary warnings about using the function with "extreme care" and I don't understand this function fully, so I'd like to ask:

Is my understanding of as_strided correct?
Is there a simple way to rewrite this so negative strides work?
Will I be able to pass a gradient w.r.t c (or r) through toeplitz_torch?

Output:
> 1. Is my understanding of as_strided correct?
The stride is an interface for your tensor to access the underlying contiguous data buffer. It does not insert values, no copies of the values are done by torch.as_strided, the strides define the artificial layout of what we refer to as multi-dimensional array (in NumPy) or tensor (in PyTorch).
As Andreas K. puts it in another answer:

Strides are the number of bytes to jump over in the memory in order to get from one item to the next item along each direction/dimension of the array. In other words, it's the byte-separation between consecutive items for each dimension.

Please feel free to read the answers over there if you have some trouble with strides. Here we will take your example and look at how it is implemented with as_strided.
The example given by Scipy for linalg.toeplitz is the following:
>>> toeplitz([1,2,3], [1,4,5,6])
array([[1, 4, 5, 6],
       [2, 1, 4, 5],
       [3, 2, 1, 4]])

To do so they first construct the list of values (what we can refer to as the underlying values, not actually underlying data): vals which is constructed as [3 2 1 4 5 6], i.e. the Toeplitz column and row flattened.
Now notice the arguments passed to np.lib.stride_tricks.as_strided:

values: vals[len(c)-1:] notice the slice: the tensors show up smaller, yet the underlying values remain, and they correspond to those of vals. Go ahead and compare the two with storage_offset: it's just an offset of 2, the values are still there! How this works is that it essentially shifts the indices such that index=0 will refer to value 1, index=1 to 4, etc...

shape: given by the column/row inputs, here (3, 4). This is the shape of the resulting object.

strides: this is the most important piece: (-n, n), in this case (-1, 1)

The most intuitive thing to do with strides is to describe a mapping between the multi-dimensional space: (i, j) ∈ [0,3[ x [0,4[ and the flattened 1D space: k ∈ [0, 3*4[. Since the strides are equal to (-n, n) = (-1, 1), the mapping is -n*i + n*j = -1*i + 1*j = j-i. Mathematically you can describe your matrix as M[i, j] = F[j-i] where F is the flattened values vector [3 2 1 4 5 6].
For instance, let's try with i=1 and j=2. If you look at the Topleitz matrix above M[1, 2] = 4. Indeed F[k] = F[j-i] = F[1] = 4
If you look closely you will see the trick behind negative strides: they allow you to 'reference' to negative indices: for instance, if you take j=0 and i=2, then you see k=-2. Remember how vals was given with an offset of 2 by slicing vals[len(c)-1:]. If you look at its own underlying data storage it's still [3 2 1 4 5 6], but has an offset. The mapping for vals (in this case i: 1D -> k: 1D) would be M'[i] = F'[k] = F'[i+2] because of the offset. This means M'[-2] = F'[0] = 3.
In the above I defined M' as vals[len(c)-1:] which basically equivalent to the following tensor:
>>> torch.as_strided(vals, size=(len(vals)-2,), stride=(1,), storage_offset=2)
tensor([1, 4, 5, 6])

Similarly, I defined F' as the flattened vector of underlying values: [3 2 1 4 5 6].
The usage of strides is indeed a very clever way to define a Toeplitz matrix!

> 2. Is there a simple way to rewrite this so negative strides work?
The issue is, negative strides are not implemented in PyTorch... I don't believe there is a way around it with torch.as_strided, otherwise it would be rather easy to extend the current implementation and provide support for that feature.
There are however alternative ways to solve the problem. It is entirely possible to construct a Toeplitz matrix in PyTorch, but that won't be with torch.as_strided.
We will do the mapping ourselves: for each element of M indexed by (i, j), we will find out the corresponding index k which is simply j-i. This can be done with ease, first by gathering all (i, j) pairs from M:
>>> i, j = torch.ones(3, 4).nonzero().T
(tensor([0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2]),
 tensor([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]))

Now we essentially have k:
>>> j-i
tensor([ 0,  1,  2,  3, -1,  0,  1,  2, -2, -1,  0,  1])

We just need to construct a flattened tensor of all possible values from the row r and column c inputs. Negative indexed values (the content of c) are put last and flipped:
>>> values = torch.cat((r, c[1:].flip(0)))
tensor([1, 4, 5, 6, 3, 2])

Finally index values with k and reshape:
>>> values[j-i].reshape(3, 4)
tensor([[1, 4, 5, 6],
        [2, 1, 4, 5],
        [3, 2, 1, 4]])

To sum it up, my proposed implementation would be:
def toeplitz(c, r):
    vals = torch.cat((r, c[1:].flip(0)))
    shape = len(c), len(r)
    i, j = torch.ones(*shape).nonzero().T
    return vals[j-i].reshape(*shape)

> 3. Will I be able to pass a gradient w.r.t c (or r) through toeplitz_torch?
That's an interesting question because torch.as_strided doesn't have a backward function implemented. This means you wouldn't have been able to backpropagate to c and r! With the above method, however, which uses 'backward-compatible' builtins, the backward pass comes free of charge.
Notice the grad_fn on the output:
>>> toeplitz(torch.tensor([1.,2.,3.], requires_grad=True),
             torch.tensor([1.,4.,5.,6.], requires_grad=True))
tensor([[1., 4., 5., 6.],
        [2., 1., 4., 5.],
        [3., 2., 1., 4.]], grad_fn=<ViewBackward>)

This was a quick draft (that did take a little while to write down), I will make some edits.  If you have some questions or remarks, don't hesitate to comment!  I would be interested in seeing other answers as I am not an expert with strides, this is just my take on the problem.


Instruction: How `make_gen_block` is defined after `__init__` in the code attached?

Input:
In the code below, self.gen is instantiated while using the make_gen_block function which is only defined later outside the __init__ attribute.
How is this possible?
Shouldn't  make_gen_block be defined before using it to instantiate self.gen so when __init__ is called, make_gen_block can be found within __init__ scope?
Thanks
class Generator(nn.Module):
    '''
    Generator Class
    Values:
        z_dim: the dimension of the noise vector, a scalar
        im_chan: the number of channels in the images, fitted for the dataset used, a scalar
              (MNIST is black-and-white, so 1 channel is your default)
        hidden_dim: the inner dimension, a scalar
    '''
    def __init__(self, z_dim=10, im_chan=1, hidden_dim=64):
        super(Generator, self).__init__()
        self.z_dim = z_dim
        # Build the neural network
        self.gen = nn.Sequential(
            self.make_gen_block(z_dim, hidden_dim * 4),
            self.make_gen_block(hidden_dim * 4, hidden_dim * 2, kernel_size=4, stride=1),
            self.make_gen_block(hidden_dim * 2, hidden_dim),
            self.make_gen_block(hidden_dim, im_chan, kernel_size=4, final_layer=True))

    def make_gen_block(self, input_channels, output_channels, kernel_size=3, stride=2, final_layer=False):
        '''
        Function to return a sequence of operations corresponding to a generator block of DCGAN,
        corresponding to a transposed convolution, a batchnorm (except for in the last layer), and an activation.
        Parameters:
            input_channels: how many channels the input feature representation has
            output_channels: how many channels the output feature representation should have
            kernel_size: the size of each convolutional filter, equivalent to (kernel_size, kernel_size)
            stride: the stride of the convolution
            final_layer: a boolean, true if it is the final layer and false otherwise
                      (affects activation and batchnorm)
        '''

        #     Steps:
        #       1) Do a transposed convolution using the given parameters.
        #       2) Do a batchnorm, except for the last layer.
        #       3) Follow each batchnorm with a ReLU activation.
        #       4) If its the final layer, use a Tanh activation after the deconvolution.

        # Build the neural block
        if not final_layer:
            return nn.Sequential(
                #### START CODE HERE ####
                nn.ConvTranspose2d(input_channels, output_channels,kernel_size,stride),
                nn.BatchNorm2d(output_channels),
                nn.ReLU())
                #### END CODE HERE ####

        else: # Final Layer
            return nn.Sequential(
                #### START CODE HERE ####
                nn.ConvTranspose2d(input_channels, output_channels,kernel_size,stride),
                #### END CODE HERE ####
                nn.Tanh())

    def unsqueeze_noise(self, noise):
        '''
        Function for completing a forward pass of the generator: Given a noise tensor,
        returns a copy of that noise with width and height = 1 and channels = z_dim.
        Parameters:
            noise: a noise tensor with dimensions (n_samples, z_dim)
        '''
        return noise.view(len(noise), self.z_dim, 1, 1)

    def forward(self, noise):
        '''
        Function for completing a forward pass of the generator: Given a noise tensor,
        returns generated images.
        Parameters:
            noise: a noise tensor with dimensions (n_samples, z_dim)
        '''
        x = self.unsqueeze_noise(noise)
        return self.gen(x)

def get_noise(n_samples, z_dim, device='cpu'):
    '''
    Function for creating noise vectors: Given the dimensions (n_samples, z_dim)
    creates a tensor of that shape filled with random numbers from the normal distribution.
    Parameters:
        n_samples: the number of samples to generate, a scalar
        z_dim: the dimension of the noise vector, a scalar
        device: the device type
    '''
    return torch.randn(n_samples, z_dim, device=device)

Output:
Note that the call to make_gen_block is actually calling self.make_gen_block. The self is important. You can see in the signature of __init__ that self is injected as the first argument. The method can be referenced because self has been passed into the __init__ method (so it is within the scope), and self is of type Generator, which has a make_gen_block method defined for it. The self instance of the class has already been constructed prior to the calling of the __init__ method.
When the class is instantiated, the __new__ method is called first, which constructs the instance and then the __init__ method is called, with the new instance injected (as self) into the method.


Instruction: does it make sense to put two convolutional layers sequentially after each other without an activation function

Input:
I am working with one model, here part of it:
...
nn.Conv2d(1, 1 * filters_multiplier, kernel_size=(3, 1)),
nn.Conv2d(1 * filters_multiplier, 6 * filters_multiplier, kernel_size=(1, 3)),

# no activation layer

nn.Conv2d(6 * filters_multiplier, 6 * filters_multiplier, kernel_size=(3, 1)),
nn.Conv2d(6 * filters_multiplier, 12 * filters_multiplier, kernel_size=(1, 3)),
nn.MaxPool2d((3, 3), stride=(2, 2)),
...

I understand that layers 1-2 and 3-4 are just convs(3,3) made for easier calculations (3x1 and 1x3 is 6 operations and 3x3 is 9). But does it make sense to put two convolutional layers sequentially after each other without an activation function or it is just a mistake? I mean between convs 1-2 and 3-4 (in commented place). I thought that a convolution followed by a convolution is just a convolution.

Output:
The purpose of adding an activation at the end of a layer is to make sure that your model can learn non-linear functions. Without activation, you will just be doing linear regression. It is the activation functions that give neural networks the power to model any function given enough depth ( Universal Approximation Theorem.
Thus it is perfectly fine not to add activations, but then you would lose on non-linearities.
If you use a linear activation function or alternatively if you don't have an activation function then no matter how many layers your neural network has, all your model is doing is just computing a linear
activation function so you might as well not have any hidden layers. (See the figure below for proof). The purpose of depth loses its essence.

For details, you can refer to this video by Andrew Ng https://youtu.be/NkOv_k7r6no.
For Convolutional Neural Network -
Well, this works the same for CNN and we don't need to prove it separately (one can do obviously). As convolutions are themselves a constraint. It just restricts the kernels to a particular spatial location, hence preserves spatial properties. A convolutional layer is really just a more restricted version of a fully connected (FC) layer. This is why you can implement convolutions using an FC layer and vice versa — they’re fundamentally the same thing. [ Source and for further details refer  - https://machinethink.net/blog/object-detection/ & https://youtu.be/bNb2fEVKeEo ]


Instruction: Adding a number to the last dimension of a tensor

Input:
I am trying to add a number to a tensor, in the way that this integer will be added as a new dimension.
The tensor is 2 rows and 7 columns:
x = [1,2,3,4,5,6,7,8,9,10,11,12,13,14]
x = torch.tensor(x)
x = x.reshape(-1,7)
print(x.shape)
print(x)

It results in:
torch.Size([2, 7])
tensor([[ 1,  2,  3,  4,  5,  6,  7],
        [ 8,  9, 10, 11, 12, 13, 14]])

The number is a float:
a= 0.19
b= torch.tensor([a])
b.reshape(-1,1)
b= b.unsqueeze(dim=1)
print(b.shape)
b

Which is:
torch.Size([1, 1])

tensor([[0.1900]])

What I want to generate is a [2,8] tensor:
tensor([[1,  2,  3,  4,  5,  6,  7,0.1900],
        [8,  9, 10, 11, 12, 13, 14,0.1900]])

So,I thought I can torch.stack to have a new dimension:
c= torch.stack((x, b), dim=-1)

Gives an error of : RuntimeError: stack expects each tensor to be equal size, but got [2, 7] at entry 0 and [1, 1] at entry 1
PS: I tried to reshape x into a shape of [14,1] and added [1,1] float tensor to make [15,1], but it added only once so I cannot make a new [2,8] anymore.
x = [1,2,3,4,5,6,7,8,9,10,11,12,13,14]
x = torch.tensor(x)
x = x.reshape(-1,1)
print(x.shape)
print(x)
torch.Size([14, 1])
tensor([[ 1],
        [ 2],
        [ 3],
        [ 4],
        [ 5],
        [ 6],
        [ 7],
        [ 8],
        [ 9],
        [10],
        [11],
        [12],
        [13],
        [14]])
print('b',b)
c= torch.cat((x, b), dim=-2)
print(c.shape)
b tensor([[0.1900]])
torch.Size([15, 1])

I would be happy to have some help!

Output:
You need to expand tensor b before concatenating them:
import torch
x = [1,2,3,4,5,6,7,8,9,10,11,12,13,14]
x = torch.tensor(x)
x = x.reshape(-1,7)
a=0.19
b= torch.tensor([a])

torch.cat((x,b.expand((2,1))),dim=1)

Will give:
tensor([[ 1.0000,  2.0000,  3.0000,  4.0000,  5.0000,  6.0000,  7.0000,  0.1900],
        [ 8.0000,  9.0000, 10.0000, 11.0000, 12.0000, 13.0000, 14.0000,  0.1900]])


Instruction: How to infer the shape of the output when connecting convolution layer with dense layers?

Input:
I am trying to construct a Convolutional Neural Network using pytorch and can not understand how to interpret the input neurons for the first densely connected layer. Say, for example, I have the following architecture:
self.conv_layer = nn.Sequential(
   nn.Conv2d(3, 32, 5),
   nn.Conv2d(32, 64, 5),
   nn.MaxPool2d(2, 2),
   nn.Conv2d(64, 128, 5),
   nn.Conv2d(128, 128, 5),
   nn.MaxPool2d(2, 2))

self.fc_layer = nn.Sequential(
   nn.Linear(X, 512),
   nn.Linear(512, 128),
   nn.Linear(128, 10))

Here X would be the number of neurons in the first linear layer. So, do I need to keep track of the shape of the output tensor at each layer so that I can figure out X?
Now, I can put the values in the formula (W - F + 2P) / S + 1 and calculate the shape after each layer, that would be somewhat convenient.
Isn't there something even more convenient which might do this automatically?

Output:
An easy solution would be to use LazyLinear layer: https://pytorch.org/docs/stable/generated/torch.nn.LazyLinear.html.
According to the documentation:

A torch.nn.Linear module where in_features is inferred ... They will be initialized after the first call to forward is done and the module will become a regular torch.nn.Linear module. The in_features argument of the Linear is inferred from the input.shape[-1].


Instruction: Swapping the batch axis has effect on the performance in pytorch?

Input:
I know that usually the batch dimension is axis zero, and I imagine this has a reason: The underlying memory for each item in the batch is contiguous.
My model calls a function that becomes simpler if I have another dimension in the first axis, so that I can use x[k] instead of x[:, k].
Results from arithmetic operations seems to keep the same memory layout
x = torch.ones(2,3,4).transpose(0,1)
y = torch.ones_like(x)
u = (x + 1)
v = (x + y)
print(x.stride(), u.stride(), v.stride())

When I create additional variables I am creating them with torch.zeros and then transposing, so that the largest stride goes to the axis 1, as well.
e.g.
a,b,c = torch.zeros(
         (3, x.shape[1], ADDITIONAL_DIM, x.shape[0]) + x.shape[2:]
).transpose(1,2)

Will create three tensors with the same batch size x.shape[1].
In terms of memory locality it would make any difference to have
a,b,c = torch.zeros(
  (x.shape[1], 3, ADDITIONAL_DIM, x.shape[0]) + x.shape[2:]
).permute(1,2,0, ...)

instead.
Should I care about this at all?

Output:
TLDR; Slices seemingly contain less information... but in fact share the identical storage buffer with the original tensor. Since permute doesn't affect the underlying memory layout, both operations are essentially equivalent.

Those two are essentially the same, the underlying data storage buffer is kept the same, only the metadata i.e. how you interact with that buffer (strides and shape) changes.
Let us look at a simple example:
>>> x = torch.ones(2,3,4).transpose(0,1)
>>> x_ptr = x.data_ptr()

>>> x.shape, x.stride(), x_ptr
(3, 2, 4), (4, 12, 1), 94674451667072

We have kept the data pointer for our 'base' tensor in x_ptr:

Slicing on the second axis:
>>> y = x[:, 0]

>>> y.shape, y.stride(), x_ptr == y.data_ptr()
(3, 4), (4, 1), True

As you can see, x and x[:, k] shared the same storage.

Permuting the first two axes then slicing on the first one:
>>> z = x.permute(1, 0, 2)[0]

>>> z.shape, z.stride(), x_ptr == z.data_ptr()
(3, 4), (4, 1), True

Here again, you notice that x.data_ptr is the same as z.data_ptr.

In fact, you can even go from y to x's representation using torch.as_strided:
>>> torch.as_strided(y, size=x.shape, stride=x.stride())
tensor([[[1., 1., 1., 1.],
         [1., 1., 1., 1.]],

        [[1., 1., 1., 1.],
         [1., 1., 1., 1.]],

        [[1., 1., 1., 1.],
         [1., 1., 1., 1.]]])

Same with z:
>>> torch.as_strided(z, size=x.shape, stride=x.stride())

Both will return a copy of x because torch.as_strided is allocating memory for the newly created tensor. These two lines were just to illustrate how we can still 'get back' to x from a slice of x, we can recover the apparent content by changing the tensor's metadata.


Instruction: HuggingFace-Transformers --- NER single sentence/sample prediction

Input:
I am trying to predict with the NER model, as in the tutorial from huggingface (it contains only the training+evaluation part).
I am following this exact tutorial here : https://github.com/huggingface/notebooks/blob/master/examples/token_classification.ipynb
The training works flawlessly, but the problems that I have begin when I try to predict on a simple sample.
model_checkpoint = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
loaded_model = AutoModel.from_pretrained('./my_model_own_custom_training.pth',
                                         from_tf=False)

input_sentence = "John Nash is a great mathematician, he lives in France"
tokenized_input_sentence = tokenizer([input_sentence],
                                     truncation=True,
                                     is_split_into_words=False,
                                     return_tensors='pt')
predictions = loaded_model(tokenized_input_sentence["input_ids"])[0]

Predictions is of shape (1,13,768)
How can I arrive at the final result of the form [JOHN <-> ‘B-PER’, … France <-> “B-LOC”], where B-PER and B-LOC are two ground truth labels, representing the tag for a person and location respectively?
The result of the prediction is:
torch.Size([1, 13, 768])

If I write:
print(predictions.argmax(axis=2))
tensor([613, 705, 244, 620, 206, 206, 206, 620, 620, 620, 477, 693, 308])

I get the tensor above.
However I would have expected to get the tensor representing the ground truth [0…8] labels from the ground truth annotations.
Summary when loading the model :

loading configuration file ./my_model_own_custom_training.pth/config.json
Model config DistilBertConfig {
“name_or_path": “distilbert-base-uncased”,
“activation”: “gelu”,
“architectures”: [
“DistilBertForTokenClassification”
],
“attention_dropout”: 0.1,
“dim”: 768,
“dropout”: 0.1,
“hidden_dim”: 3072,
“id2label”: {
“0”: “LABEL_0”,
“1”: “LABEL_1”,
“2”: “LABEL_2”,
“3”: “LABEL_3”,
“4”: “LABEL_4”,
“5”: “LABEL_5”,
“6”: “LABEL_6”,
“7”: “LABEL_7”,
“8”: “LABEL_8”
},
“initializer_range”: 0.02,
“label2id”: {
“LABEL_0”: 0,
“LABEL_1”: 1,
“LABEL_2”: 2,
“LABEL_3”: 3,
“LABEL_4”: 4,
“LABEL_5”: 5,
“LABEL_6”: 6,
“LABEL_7”: 7,
“LABEL_8”: 8
},
“max_position_embeddings”: 512,
“model_type”: “distilbert”,
“n_heads”: 12,
“n_layers”: 6,
“pad_token_id”: 0,
“qa_dropout”: 0.1,
“seq_classif_dropout”: 0.2,
“sinusoidal_pos_embds”: false,
"tie_weights”: true,
“transformers_version”: “4.8.1”,
“vocab_size”: 30522
}

Output:
The answer is a bit trickier than expected[Huge credits to Niels Rogge].
Firstly, loading models in huggingface-transformers can be done in (at least) two ways:

AutoModel.from_pretrained('./my_model_own_custom_training.pth', from_tf=False)
AutoModelForTokenClassification.from_pretrained('./my_model_own_custom_training.pth', from_tf=False)

It seems that, according to the task at hand, different AutoModels subclasses need to be used. In this scenario I posted, it is the AutoModelForTokenClassification() that has to be used.
After that, a solution to obtain the predictions would be to do the following:
# forward pass
outputs = model(**encoding)
logits = outputs.logits

predictions = logits.argmax(-1)


Instruction: Does packing BooleanTensor's to ByteTensor's affect training of LSTM (or other ML models)?

Input:
I am working on an LSTM to generate music. My input data will be a BooleanTensor of size 88xLx3, 88 being the amount of available notes, L being the length of each "piece" which will be in the order of 1k - 10k (TBD), and 3 being the parts for "lead melody", "accompaniment", and "bass". A value of 0 would symbolize that that specific note is not being played by that part (instrument) at that time, and a 1 would symbolize that it is.
The problem is that each entry of a BooleanTensor takes 1 byte of space in memory instead of 1 bit, which wastes a lot of valuable GPU memory.
As a solution I thought of packing each BooleanTensor to a ByteTensor (uint8) of size 11xLx3 or 88x(L/8)x3.
My question is: Would packing the data as such have an effect on the learning and generation of the LSTM or would the ByteTensor-based data and model be equivalent to their BooleanTensor-based counterparts in practice?

Output:
I wouldn't really care about the fact that the input is taking X instead of Y number of bits, at least when it comes to GPU memory. Most of it is occupied by the network's weights and intermediate outputs, which will likely be float32 anyway (maybe float16). There is active research on training with lower precision (even binary training), but based on your question, it seems completely unnecessary. Lastly, you can always try Quantization to your production models, if you really need it.
With regards to the packing: it can have an impact, especially if you do it naively. The grouping you're suggesting doesn't seem to be a natural one, therefore it may be harder to learn patterns from the grouped data than otherwise. There'll always be workarounds, but then this answer become an opinion because it is almost impossible to antecipate what could work; an opinion-based questions/answer are off-topic around here :)


Instruction: Why tensor size was not changed?

Input:
I made the toy CNN model.
class Test(nn.Module):
    def __init__(self):
        super(Test, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(3,300,3),
            nn.Conv2d(300,500,3),
            nn.Conv2d(500,1000,3),
        )
        self.fc = nn.Linear(3364000,1)

    def forward(self, x):
        out = self.conv(x)
        out = out.view(out.size(0), -1)
        out = self.fc(out)
        return out

Then, I had checked model.summary via this code
model = Test()
model.to('cuda')
for param in model.parameters():
    print(param.dtype)
    break
summary_(model, (3,64,64))

And I was able to get the following results:
torch.float32
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Conv2d-1          [-1, 300, 62, 62]           8,400
            Conv2d-2          [-1, 500, 60, 60]       1,350,500
            Conv2d-3         [-1, 1000, 58, 58]       4,501,000
            Linear-4                    [-1, 1]       3,364,001
================================================================
Total params: 9,223,901
Trainable params: 9,223,901
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.05
Forward/backward pass size (MB): 48.20
Params size (MB): 35.19
Estimated Total Size (MB): 83.43
----------------------------------------------------------------

I want to reduce model size cuz i wanna increase the batch size.
So, I had changed torch.float32 -> torch.float16 via NVIDIA/apex
model = Test()
model.to('cuda')
opt_level = 'O3'
optimizer = optim.Adam(model.parameters(), lr=0.001)
model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level)
for param in model.parameters():
    print(param.dtype)
    break
summary_(model, (3,64,64))

Selected optimization level O3:  Pure FP16 training.
Defaults for this optimization level are:
enabled                : True
opt_level              : O3
cast_model_type        : torch.float16
patch_torch_functions  : False
keep_batchnorm_fp32    : False
master_weights         : False
loss_scale             : 1.0
Processing user overrides (additional kwargs that are not None)...
After processing overrides, optimization options are:
enabled                : True
opt_level              : O3
cast_model_type        : torch.float16
patch_torch_functions  : False
keep_batchnorm_fp32    : False
master_weights         : False
loss_scale             : 1.0
torch.float16
----------------------------------------------------------------
        Layer (type)               Output Shape         Param #
================================================================
            Conv2d-1          [-1, 300, 62, 62]           8,400
            Conv2d-2          [-1, 500, 60, 60]       1,350,500
            Conv2d-3         [-1, 1000, 58, 58]       4,501,000
            Linear-4                    [-1, 1]       3,364,001
================================================================
Total params: 9,223,901
Trainable params: 9,223,901
Non-trainable params: 0
----------------------------------------------------------------
Input size (MB): 0.05
Forward/backward pass size (MB): 48.20
Params size (MB): 35.19
Estimated Total Size (MB): 83.43
----------------------------------------------------------------

As a result, torch.dtype was changed torch.float16 from torch.float32.
But, Param size (MB): 35.19 was not changed.
Why happen this? plz tell me about this.
Thanks.

Output:
Mixed precision does not mean that your model becomes half original size. The parameters remain in float32 dtype by default and they are cast to float16 automatically during certain operations of the neural network training. This is applicable to input data as well.
The torch.cuda.amp provides the functionality to perform this automatic conversion from float32 to float16 during certain operations of training like Convolutions. Your model size will remain the same. Reducing model size is called quantization and it is different than mixed-precision training.
You can read to more about mixed-precision training at NVIDIA's blog and Pytorch's blog.


Instruction: How can I simulate this paragraph about changing the learning rate picked from Resnet paper?

Input:
While going through this paper, I found they are changing learning rate in between of training/validation iterations.

We start with a learning rate of 0.1, divide it by 10 at 32k and 48k iterations, and terminate training at 64k iterations, which is determined on a 45k/5k train/val split.

Output:
Generally, You're probably looking for
torch.optim.lr_scheduler
Specifically, You can implement the reduction of learning_rate after n epochs using lr_scheduler.MultiStepLR
Following the advised way to use lr_schedulers, You will need to recalculate the milestones from steps to epochs as the updates are done after the whole epochs rather than steps.
If that won't give You the satisfactory result, You can actually "cheat" by calling
scheduler.step() after each batch (step) and then passing the milestones in number of steps.
Remember then not to confuse yourself or the ones that will happen to edit Your code some day, at least put a comment to indicate that You're using the library function in a little less obvious way :D


Instruction: Use torch.square() inside a nn.Sequential layer in PyTorch

Input:
I want to square the result of a maxpool layer.
I tried the following:
class CNNClassifier(Classifier):  # nn.Module
    def __init__(self, in_channels):
        super().__init__()
        self.save_hyperparameters('in_channels')
        self.cnn = nn.Sequential(
            # maxpool
            nn.MaxPool2d((1, 5), stride=(1, 5)),
            torch.square(),
            # layer1
            nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=5,
        )

Which to the experienced PyTorch user for sure makes no sense.
Indeed, the error is quite clear:

TypeError: square() missing 1 required positional arguments: "input"

How can I feed in to square the tensor from the preceding layer?

Output:
You can't put a PyTorch function in a nn.Sequential pipeline, it needs to be a nn.Module.
You could wrap it like this:
class Square(nn.Module):
    def forward(self, x):
        return torch.square(x)

Then use it inside your sequential layer like so:
class CNNClassifier(Classifier):  # nn.Module
    def __init__(self, in_channels):
        super().__init__()
        self.save_hyperparameters('in_channels')
        self.cnn = nn.Sequential(
            nn.MaxPool2d((1, 5), stride=(1, 5)),
            Square(),
            nn.Conv2d(in_channels=in_channels, out_channels=32, kernel_size=5))


Instruction: How to find the name of layers in preloaded torchvision models?

Input:
I'm trying to use GradCAM with a Deeplabv3 resnet50 model preloaded from torchvision, but in Captum I need to say the name of the layer (of type nn.module). I can't find any documentation for how this is done, does anyone possibly have any ideas of how to get the name of the final ReLu layer?
Thanks in advance!

Output:
You can have a look at its representation and get an idea of where it's located by simply printing it:
>>> model = torchvision.models.segmentation.deeplabv3_resnet50()
>>> model
DeepLabV3(
  (backbone): IntermediateLayerGetter(
    (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (relu): ReLU(inplace=True)
    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (layer1): Sequential(
      (0): Bottleneck(
        ...

To get the actual exact name of the layer you can loop over the modules with named_modules and only pick the nn.ReLU layers:
>>> relus = [name for name, module in model.named_modules() if isinstance(module, nn.ReLU)]

>>> relus
['backbone.relu',
 'backbone.layer1.0.relu',
 'backbone.layer1.1.relu',
 'backbone.layer1.2.relu',
 'backbone.layer2.0.relu',
 'backbone.layer2.1.relu',
 'backbone.layer2.2.relu',
 'backbone.layer2.3.relu',
 'backbone.layer3.0.relu',
 'backbone.layer3.1.relu',
 'backbone.layer3.2.relu',
 'backbone.layer3.3.relu',
 'backbone.layer3.4.relu',
 'backbone.layer3.5.relu',
 'backbone.layer4.0.relu',
 'backbone.layer4.1.relu',
 'backbone.layer4.2.relu',
 'classifier.0.convs.0.2',
 'classifier.0.convs.1.2',
 'classifier.0.convs.2.2',
 'classifier.0.convs.3.2',
 'classifier.0.convs.4.3',
 'classifier.0.project.2',
 'classifier.3']

Then pick the last one:
>>> relus[-1]
'classifier.3'


Instruction: Problem with Loading tiny imagenet via torch DataLoader

Input:
I'm using tiny-imagenet-200 and I'm not sure that loading them with torch.utils.data.DataLoader is possible or not.
I downloaded tiny-imagenet-200 from Stanford site, but the format of validation set in a directory with name val_0 to val_9999 and the label of them is in a .txt.
How can I load this directory via torch.utils.data.DataLoader?
I tried:
datasets.ImageFolder(args.val_dir, transforms.Compose([
            OpencvResize(256),
            transforms.CenterCrop(224),
            ToBGRTensor(),
        ])

but it doesn't work.

Output:
You can't do that using ImageFolder directly. There are alternatives, though:

You can read the annotation file and re-structure the directories to enable the usage of ImageFolder, as in here;
You can implement a custom Dataset. Luckly, as Tiny ImageNet is a popular dataset, you can find many implementations online. For instance, this one.


Instruction: multiclass sequence classifiaction with fastai and huggingface

Input:
I am looking to implement DistilBERT via fastai and huggingface for a mutliclass sequence classification problem. I found a useful tutorial that gave a good example on how to do this with binary classification. The code is below:
# !pip install torch==1.9.0
# !pip install torchtext==0.10
# !pip install transformers==4.7
# !pip install fastai==2.4

from fastai.text.all import *
from sklearn.model_selection import train_test_split
import pandas as pd
import glob
from transformers import AutoTokenizer, AutoModelForSequenceClassification

hf_tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
hf_model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased")

"""
train_df and val_df looks like this:

      label text
4240    5   whoa interesting.
13      7   you could you could we just
4639    4   you set the goal,
28      1   because ive already agreed to that
66      8   oh hey freshman thats you gona need
"""

print(list(train_df.label.value_counts().index))
"""
[4, 1, 5, 6, 7, 0, 2, 3, 8]
"""

class HF_Dataset(torch.utils.data.Dataset):
    def __init__(self, df, hf_tokenizer):
        self.df = df
        self.hf_tokenizer = hf_tokenizer

        self.label_map = {
            0:0,
            1:0,
            2:0,
            3:0,
            4:1,
            5:1,
            6:1,
            7:1,
            8:1
        }

    def __len__(self):
        return len(self.df)

    def decode(self, token_ids):
        return ' '.join([hf_tokenizer.decode(x) for x in tokenizer_outputs['input_ids']])

    def decode_to_original(self, token_ids):
        return self.hf_tokenizer.decode(token_ids.squeeze())

    def __getitem__(self, index):
        label, text = self.df.iloc[index]
        label = self.label_map[label]
        label = torch.tensor(label)

        tokenizer_output = self.hf_tokenizer(text, return_tensors="pt", padding='max_length', truncation=True, max_length=512)

        tokenizer_output['input_ids'].squeeze_()
        tokenizer_output['attention_mask'].squeeze_()

        return tokenizer_output, label

train_dataset = HF_Dataset(train_df, hf_tokenizer)
valid_dataset = HF_Dataset(valid_df, hf_tokenizer)

train_dl = DataLoader(train_dataset, bs=16, shuffle=True)
valid_dl = DataLoader(valid_dataset, bs=16)
dls = DataLoaders(train_dl, valid_dl)
hf_model(**batched_data)

class HF_Model(nn.Module):

    def __init__(self, hf_model):
        super().__init__()

        self.hf_model = hf_model

    def forward(self, tokenizer_outputs):

        model_output = self.hf_model(**tokenizer_outputs)

        return model_output.logits

model = HF_Model(hf_model)
# Manually popping the model onto the gpu since the data is in a dictionary format
# (doesn't automatically place model + data on gpu otherwise)
learn = Learner(dls, model, loss_func=nn.CrossEntropyLoss(), metrics=[accuracy])
learn.fit_one_cycle(3, 1e-4)

This works fine. However, I mapped my multiclass labels to 2 labels to allow this to work. I actually have 9 classes. I tried adjusting the label mapping scheme in HF_Dataset() class to match my actual labels like below:
class HF_Dataset(torch.utils.data.Dataset):
    def __init__(self, df, hf_tokenizer):
        self.df = df
        self.hf_tokenizer = hf_tokenizer

        self.label_map = {
            0:0,
            1:1,
            2:2,
            3:3,
            4:4,
            5:5,
            6:6,
            7:7,
            8:8
        }

    def __len__(self):
        return len(self.df)

    def decode(self, token_ids):
        return ' '.join([hf_tokenizer.decode(x) for x in tokenizer_outputs['input_ids']])

    def decode_to_original(self, token_ids):
        return self.hf_tokenizer.decode(token_ids.squeeze())

    def __getitem__(self, index):
        label, text = self.df.iloc[index]
        label = self.label_map[label]
        label = torch.tensor(label)

        tokenizer_output = self.hf_tokenizer(text, return_tensors="pt", padding='max_length', truncation=True, max_length=512)

        tokenizer_output['input_ids'].squeeze_()
        tokenizer_output['attention_mask'].squeeze_()

        return tokenizer_output, label

Every line works until learn.fit_one_cycle.
Here is the full stack trace from this line:

 0.00% [0/3 00:00<00:00]
epoch   train_loss  valid_loss  accuracy    time

 0.00% [0/519 00:00<00:00]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-21-0ec2ff9e12e1> in <module>
----> 1 learn.fit_one_cycle(3, 1e-4)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/callback/schedule.py in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
    111     scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
    112               'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 113     self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)
    114
    115 # Cell

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in fit(self, n_epoch, lr, wd, cbs, reset_opt)
    219             self.opt.set_hypers(lr=self.lr if lr is None else lr)
    220             self.n_epoch = n_epoch
--> 221             self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)
    222
    223     def _end_cleanup(self): self.dl,self.xb,self.yb,self.pred,self.loss = None,(None,),(None,),None,None

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    161
    162     def _with_events(self, f, event_type, ex, final=noop):
--> 163         try: self(f'before_{event_type}');  f()
    164         except ex: self(f'after_cancel_{event_type}')
    165         self(f'after_{event_type}');  final()

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _do_fit(self)
    210         for epoch in range(self.n_epoch):
    211             self.epoch=epoch
--> 212             self._with_events(self._do_epoch, 'epoch', CancelEpochException)
    213
    214     def fit(self, n_epoch, lr=None, wd=None, cbs=None, reset_opt=False):

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    161
    162     def _with_events(self, f, event_type, ex, final=noop):
--> 163         try: self(f'before_{event_type}');  f()
    164         except ex: self(f'after_cancel_{event_type}')
    165         self(f'after_{event_type}');  final()

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _do_epoch(self)
    204
    205     def _do_epoch(self):
--> 206         self._do_epoch_train()
    207         self._do_epoch_validate()
    208

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _do_epoch_train(self)
    196     def _do_epoch_train(self):
    197         self.dl = self.dls.train
--> 198         self._with_events(self.all_batches, 'train', CancelTrainException)
    199
    200     def _do_epoch_validate(self, ds_idx=1, dl=None):

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    161
    162     def _with_events(self, f, event_type, ex, final=noop):
--> 163         try: self(f'before_{event_type}');  f()
    164         except ex: self(f'after_cancel_{event_type}')
    165         self(f'after_{event_type}');  final()

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in all_batches(self)
    167     def all_batches(self):
    168         self.n_iter = len(self.dl)
--> 169         for o in enumerate(self.dl): self.one_batch(*o)
    170
    171     def _do_one_batch(self):

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in one_batch(self, i, b)
    192         b = self._set_device(b)
    193         self._split(b)
--> 194         self._with_events(self._do_one_batch, 'batch', CancelBatchException)
    195
    196     def _do_epoch_train(self):

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _with_events(self, f, event_type, ex, final)
    161
    162     def _with_events(self, f, event_type, ex, final=noop):
--> 163         try: self(f'before_{event_type}');  f()
    164         except ex: self(f'after_cancel_{event_type}')
    165         self(f'after_{event_type}');  final()

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/fastai/learner.py in _do_one_batch(self)
    173         self('after_pred')
    174         if len(self.yb):
--> 175             self.loss_grad = self.loss_func(self.pred, *self.yb)
    176             self.loss = self.loss_grad.clone()
    177         self('after_loss')

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
   1119     def forward(self, input: Tensor, target: Tensor) -> Tensor:
   1120         return F.cross_entropy(input, target, weight=self.weight,
-> 1121                                ignore_index=self.ignore_index, reduction=self.reduction)
   1122
   1123

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   2822     if size_average is not None or reduce is not None:
   2823         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2824     return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2825
   2826

IndexError: Target 6 is out of bounds.

This seems like it should be a simple fix. Do I need to adjust something in the model architecture to allow it to accept 9 labels? Or do I need to one hot encode my labels? If so, is there a solution prebuilt to do this in the pipeline?

Output:
You need to define num_labels=9 when loading the model:
hf_model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=9)

The default value is 2, which suits the first use-case, but breaks when you tried to change.
Note that the lib explictly says that the classifier (which generates the .logits that are of your interest) is randomly initialized:

Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.weight', 'pre_classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.


Instruction: Pytorch ConvNet loss remains unchanged and only one class is predicted

Input:
My ConvNet is only predicting a single class and the loss remains unchanged.
I have tried the following:

added class weights to be proportional to the data sizes (1-(class occurrences/total data))
adjusted learning rate attempting to find a sweet spot
adjusted gamma (Multiplicative factor of learning rate decay)
used gamma in the loss function, and not used it in loss function (Mainly have been experimenting with Adam)
Tried a much more complex ConvNet than the 'Simple' one currently in use

I am unsure of where to go from here. It seems no matter what I try the Neural Net always predicts the same class (I have been able to get it to predict the other class by throwing the weights very out of proportion)
Below is the output from running my program. It should have all the relevant information in it to come up with some ideas as to how to fix it. If you need to see some of the source code or are curious about what the dataset looks like, please ask.
Any help is greatly appreciated. I have been stumped on this issue for quite a while now. Thank you!
Train dataset length: 27569
Test dataset length: 4866

Image preprocessing:
None
Input dimensions: 28 X 28
Output dimension: 2

Model: Simple
NeuralNetwork(
  (flatten): Flatten(start_dim=1, end_dim=-1)
  (linear_relu_stack): Sequential(
    (0): Linear(in_features=784, out_features=512, bias=True)
    (1): ReLU()
    (2): Linear(in_features=512, out_features=512, bias=True)
    (3): ReLU()
    (4): Linear(in_features=512, out_features=2, bias=True)
    (5): ReLU()
  )
)

Optimizer: Adam
Learning rate: 0.0001

Loss function: CEL
class weights: tensor([0.3481, 0.6519], device='cuda:0')
Multiplicative factor of learning rate decay: 0.0005

Train Epoch: 1 [0/27569 (0%)]   Loss: 3785.907959
Train Epoch: 1 [6400/27569 (23%)]   Loss: 0.693147
Train Epoch: 1 [12800/27569 (46%)]  Loss: 0.693147
Train Epoch: 1 [19200/27569 (70%)]  Loss: 0.693147
Train Epoch: 1 [25600/27569 (93%)]  Loss: 0.693147

Test set: Average loss: 0.0110, Accuracy: 3172/4866 (65%)

actual count: [3172, 1694]
predicted count: [4866, 0]

Output:
It is very uncommon to have a ReLU after the last Linear layer (where the logits come from). Consider removing it.
In addition, maybe your learning rate is too high. You could try tweaking it a little bit. Check if the loss decreases smoothly between the iterations (which is ideal in most cases), otherwise decrease it


Instruction: Implementing Early Stopping in Pytorch without Torchsample

Input:
As a Pytorch newbie (coming from tensorflow), I am unsure of how to implement Early Stopping. My research has led me discover that pytorch does not have a native way to so this. I have also discovered torchsample, but am unable to install it in my conda environment for whatever reason. Is there a simple way to go about applying early stopping without it? Here is my current setup:
class RegressionDataset(Dataset):

    def __init__(self, X_data, y_data):
        self.X_data = X_data
        self.y_data = y_data

    def __getitem__(self, index):
        return self.X_data[index], self.y_data[index]

    def __len__(self):
        return len(self.X_data)

train_dataset = RegressionDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train).float())
val_dataset = RegressionDataset(torch.from_numpy(X_val).float(), torch.from_numpy(y_val).float())
test_dataset = RegressionDataset(torch.from_numpy(X_test).float(), torch.from_numpy(y_test).float())

# Model Params
EPOCHS = 100
BATCH_SIZE = 1000
LEARNING_RATE = 0.001
NUM_FEATURES = np.shape(X_test)[1]

# Initialize Dataloader
train_loader = DataLoader(dataset = train_dataset, batch_size=BATCH_SIZE, shuffle = True)
val_loader = DataLoader(dataset = val_dataset, batch_size=BATCH_SIZE)
test_loader = DataLoader(dataset = test_dataset, batch_size=BATCH_SIZE)

# Define Neural Network Architecture
class MultipleRegression(nn.Module):
    def __init__(self, num_features):
        super(MultipleRegression, self).__init__()

        # Define architecture
        self.layer_1 = nn.Linear(num_features, 16)
        self.layer_2 = nn.Linear(16, 32)
        self.layer_3 = nn.Linear(32, 25)
        self.layer_4 = nn.Linear(25, 20)
        self.layer_5 = nn.Linear(20, 16)
        self.layer_out = nn.Linear(16, 1)

        self.relu = nn.ReLU() # ReLU applied to all layers

        # Initialize weights and biases
        nn.init.xavier_uniform_(self.layer_1.weight)
        nn.init.zeros_(self.layer_1.bias)
        nn.init.xavier_uniform_(self.layer_2.weight)
        nn.init.zeros_(self.layer_2.bias)
        nn.init.xavier_uniform_(self.layer_3.weight)
        nn.init.zeros_(self.layer_3.bias)
        nn.init.xavier_uniform_(self.layer_4.weight)
        nn.init.zeros_(self.layer_4.bias)
        nn.init.xavier_uniform_(self.layer_5.weight)
        nn.init.zeros_(self.layer_5.bias)
        nn.init.xavier_uniform_(self.layer_out.weight)
        nn.init.zeros_(self.layer_out.bias)

    def forward(self, inputs):
        x = self.relu(self.layer_1(inputs))
        x = self.relu(self.layer_2(x))
        x = self.relu(self.layer_3(x))
        x = self.relu(self.layer_4(x))
        x = self.relu(self.layer_5(x))
        x = self.layer_out(x)
        return(x)

    def predict(self, test_inputs):
        x = self.relu(self.layer_1(test_inputs))
        x = self.relu(self.layer_2(x))
        x = self.relu(self.layer_3(x))
        x = self.relu(self.layer_4(x))
        x = self.relu(self.layer_5(x))
        x = self.layer_out(x)
        return(x)

# Check for GPU
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)

model = MultipleRegression(NUM_FEATURES)
model.to(device)
print(model)
criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr = LEARNING_RATE)

# define dictionary to store loss/epochs for training and validation
loss_stats = {
    "train": [],
    "val": []
    }

# begin training
print("Begin Training")
for e in tqdm(range(1, EPOCHS+1)):
    # Training
    train_epoch_loss = 0
    model.train()

    for X_train_batch, y_train_batch in train_loader:
        X_train_batch, y_train_batch = X_train_batch.to(device), y_train_batch.to(device)
        optimizer.zero_grad()

        y_train_pred = model(X_train_batch)

        train_loss = criterion(y_train_pred, y_train_batch.unsqueeze(1))

        train_loss.backward()
        optimizer.step()

        train_epoch_loss += train_loss.item()

        # validation
        with torch.no_grad():

            val_epoch_loss = 0

            model.eval()
            for X_val_batch, y_val_batch in val_loader:
                X_val_batch, y_val_batch = X_val_batch.to(device), y_val_batch.to(device)

                y_val_pred = model(X_val_batch)

                val_loss = criterion(y_val_pred, y_val_batch.unsqueeze(1))

                val_epoch_loss += val_loss.item()
    loss_stats["train"].append(train_epoch_loss/len(train_loader))
    loss_stats["val"].append(val_epoch_loss/len(val_loader))
    print(f"Epoch {e}: \ Train loss: {train_epoch_loss/len(train_loader):.5f} \ Val loss: {val_epoch_loss/len(val_loader):.5f}")

# Visualize loss and accuracy
train_val_loss_df = pd.DataFrame.from_dict(loss_stats).reset_index().melt(id_vars=["index"]).rename(columns = {"index":"epochs"})
plt.figure()
sns.lineplot(data = train_val_loss_df, x = "epochs", y = "value", hue = "variable").set_title("Train-Val Loss/Epoch")

# Test model
y_pred_list = []
with torch.no_grad():
    model.eval()
    for X_batch, _ in test_loader:
        X_batch = X_batch.to(device)
        y_test_pred = model(X_batch)
        y_pred_list.append(y_test_pred.cpu().numpy())
y_pred_list = [a.squeeze().tolist() for a in y_pred_list]

y_pred_list = [item for sublist in y_pred_list for item in sublist]
y_pred_list = np.array(y_pred_list)

mse = mean_squared_error(y_test, y_pred_list)
r_square = r2_score(y_test, y_pred_list)
print("Mean Squared Error :", mse)
print("R^2 :", r_square)

Output:
A basic way to do this is to keep track of the best validation loss obtained so far.
You can have a variable best_loss = 0 initialized before your loop over epochs (or you could do other things like best loss per epoch, etc.).
After each validation pass then do:
if val_loss > best_loss:
    best_loss = val_loss
    # At this point also save a snapshot of the current model
    torch.save(model, 'my_model_best_loss.pth')

Then, if the best_loss does not improve significantly after some number of training steps, or by the end of the epoch, or if it val_loss gets worse, break out of the loop and terminate the training there.
For implementing algorithms like early stopping (and your training loop in general) you may find it easier to give PyTorch Lightning a try (no affiliation, but it's much easier than trying to roll everything by hand).


Instruction: How to apply mask to image tensors in PyTorch?

Input:
Applying mask with NumPy or OpenCV is a relatively straightforward process. However, if I need to use masked image in loss calculations of my optimization algorithm, I need to employ exclusively PyTorch, as doing otherwise interferes with gradient computations.
Assuming that I have an image tensor [1, 512, 512, 3] (batch, height, width, channels) and a mask tensor [1, 20, 512, 512] (batch, channels, height, width) where every channel corresponds to one of 20 segmentation classes, I want to get a masked image tensor that fills every pixel with black (0, 0, 0), except for those belonging to one or more specified segmentation classes.
Here is how it can be done with numpy:
import numpy as np
import torch

# Create dummy image and mask
image_tensor = torch.randn([1, 512, 512, 3])
mask_tensor = torch.randn([1, 20, 512, 512])

# Apply argmax to mask
mask_tensor =  torch.max(mask_tensor, 1)[1] # -> 1, 512, 512

# Define mask function
def selective_mask(image_src, mask, dims=[]):
    h, w = mask.shape
    background = np.zeros([h, w, 3], dtype=np.uint8)
    for j_, j in enumerate(mask[:, :]):
        for k_, k in enumerate(j):
            if k in dims:
                background[j_, k_] = image_src[j_, k_]
    output = background
    return output

# Convert tensors to numpy:
image = image_tensor.squeeze(0).cpu().numpy()
mask = mask_tensor.squeeze(0).cpu().nmpy()

# Apply mask function for several classes
image_masked = selective_mask(image, mask, dims=[5, 6, 8])

How should my code be changed to bring it in line with the PyTorch requirements?

Output:
First of all, the definition of the function selective_mask is far for what You may call 'straightforward'. The key point in using numpy (and torch, which is designed to be mostly compatible) is to take advantage of the vectorization of operations and to avoid using loops, which are not parallelizable.
If You rewrite the said function in this manner:
def selective_mask(image_src, mask, channels=[]):
    mask = mask[np.array(channels).astype(int)]
    return np.sign(np.sum(mask, axis=0), dtype=image_src.dtype) * image_src

it will turn out that You can actually do the same with pytorch tensors (here no need to squeeze the batch (first) dimension):
def selective_mask_t(image_src, mask, channels=[]):
    mask = mask[:, torch.tensor(channels).long()]
    mask = torch.sgn(torch.sum(mask, dim=1)).to(dtype=image_src.dtype).unsqueeze(-1)
    return mask * image_src

Also, You probably want to produce the mask itself this way:
(BTW using a combination of max and sgn here should actually work faster than setting elements indexed by argmax)
# Create dummy image and mask
image_tensor = torch.randn([1, 512, 512, 3])
mask_tensor = torch.randn([1, 20, 512, 512])

# Discreticize the mask (set to one in the channel with the highest value) -> 1, 20, 512, 512
mask_tensor = torch.sgn(mask_tensor - torch.max(mask_tensor, 1)[0].unsqueeze(1)) + 1.

Then it should work just fine:
print(selective_mask_t(image_tensor, mask_tensor, [5, 6, 8]))


Instruction: How to fix ValueError: too many values to unpack (expected 2)?

Input:
Recently faced with such a problem: ValueError: too many values to unpack (expected 2).
import os
import natsort
from PIL import Image
import torchvision
import torch
import torch.optim as optim
from torchvision import transforms, models
from torch.utils.data import DataLoader, Dataset
import torch.nn as nn
import torch.nn.functional as F

root_dir = './images/'

class Col(Dataset):
    def __init__(self, main_dir, transform):
        self.main_dir = main_dir
        self.transform = transform
        all_images = self.all_img(main_dir = main_dir)
        self.total_imges = natsort.natsorted(all_images)

    def __len__(self):
        return len(self.total_imges)

    def __getitem__(self, idx):
        img_loc = os.path.join(self.total_imges[idx])
        image = Image.open(img_loc).convert("RGB")
        tensor_image = self.transform(image)
        return tensor_image

    def all_img(self, main_dir):
        img = []
        for path, subdirs, files in os.walk(main_dir):
            for name in files:
                img.append(os.path.join(path, name))
        return img

model = models.resnet18(pretrained=False)
model.fc = nn.Sequential(nn.Linear(model.fc.in_features, 256),
                         nn.ReLU(),
                         nn.Dropout(p=0.3),
                         nn.Linear(256, 100),
                         nn.ReLU(),
                         nn.Dropout(p=0.4),
                         nn.Linear(100,9))
# model.load_state_dict(torch.load('model.pth'))

for name, param in model.named_parameters():
    if("bn" not in name):
        param.requires_grad = False

transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.5457, 0.5457, 0.5457], std=[0.2342, 0.2342, 0.2342])
])

data = Col(main_dir=root_dir, transform=transform)

dataset = torch.utils.data.DataLoader(data, batch_size=130)

train_set, validate_set= torch.utils.data.random_split(dataset, [round(len(dataset)*0.7), (len(dataset) - round(len(dataset)*0.7))])

if torch.cuda.is_available():
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

model.to(device)

def train(model, optimizer, loss_fn, train_set, validate_set, epochs=20, device="cpu"):
    for epoch in range(1, epochs+1):
        training_loss = 0.0
        valid_loss = 0.0
        model.train()
        for batch in train_set:
            optimizer.zero_grad()
            inputs, targets = batch
            inputs = inputs.to(device)
            targets = targets.to(device)
            output = model(inputs)
            loss = loss_fn(output, targets)
            loss.backward()
            optimizer.step()
            training_loss += loss.data.item() * inputs.size(0)
        training_loss /= len(train_set.dataset)

        model.eval()
        num_correct = 0
        num_examples = 0
        for batch in validate_set:
            inputs, targets = batch
            inputs = inputs.to(device)
            output = model(inputs)
            targets = targets.to(device)
            loss = loss_fn(output,targets)
            valid_loss += loss.data.item() * inputs.size(0)
            correct = torch.eq(torch.max(F.softmax(output, dim=1), dim=1)[1], targets)
            num_correct += torch.sum(correct).item()
            num_examples += correct.shape[0]
        valid_loss /= len(validate_set.dataset)

        print('Epoch: {}, Training Loss: {:.2f}, Validation Loss: {:.2f}, accuracy = {:.2f}'.format(epoch, training_loss,
        valid_loss, num_correct / num_examples))

optimizer = optim.Adam(model.parameters(), lr=0.0001)

But the call to this function
train(model, optimizer,torch.nn.CrossEntropyLoss(), train_set.dataset, validate_set.dataset, epochs=100, device=device)

gives this error
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/tmp/ipykernel_4828/634509595.py in <module>
----> 1 train(model, optimizer,torch.nn.CrossEntropyLoss(), train_set.dataset, validate_set.dataset, epochs=100, device=device)

/tmp/ipykernel_4828/1473922939.py in train(model, optimizer, loss_fn, train_set, validate_set, epochs, device)
      6         for batch in train_set:
      7             optimizer.zero_grad()
----> 8             inputs, targets = batch
      9             inputs = inputs.to(device)
     10             targets = targets.to(device)

ValueError: too many values to unpack (expected 2)

Output: Batch doesn't contain both the inputs and the targets. Your problem is just that getitem returns only tensor_image (which is presumably the inputs) and not whatever targets should be.


Instruction: TensorFlow vs PyTorch: Memory usage

Input:
I have PyTorch 1.9.0 and TensorFlow 2.6.0 in the same environment, and both recognizing the all GPUs.
I was comparing the performance of both, so I did this small simple test, multiplying large matrices (A and B, both 2000x2000) several times (10000x):
import numpy as np
import os
import time

def mul_torch(A,B):
    # PyTorch matrix multiplication
    os.environ['KMP_DUPLICATE_LIB_OK']='True'
    import torch
    A, B = torch.Tensor(A.copy()), torch.Tensor(B.copy())
    A = A.cuda()
    B = B.cuda()
    start = time.time()
    for i in range(10000):
        C = torch.matmul(A, B)
    torch.cuda.empty_cache()
    print('PyTorch:', time.time() - start, 's')

    return C

def mul_tf(A,B):
    # TensorFlow Matrix Multiplication
    import tensorflow as tf
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
    with tf.device('GPU:0'):
        A = tf.constant(A.copy())
        B = tf.constant(B.copy())
    start = time.time()
    for i in range(10000):
        C = tf.math.multiply(A, B)
    print('TensorFlow:', time.time() - start, 's')
    return C

if __name__ == '__main__':
    A = np.load('A.npy')
    B = np.load('B.npy')

    n = 2000
    A = np.random.rand(n, n)
    B = np.random.rand(n, n)

    PT = mul_torch(A, B)
    time.sleep(5)
    TF = mul_tf(A, B)

As a result:
PyTorch: 19.86856198310852 s
TensorFlow: 2.8338065147399902 s

I was not expecting these results, I thought the results should be similar.
Investigating the GPU performance, I noticed that both are using GPU full capacity, but PyTorch uses a small fraction of the memory Tensorflow uses. It explains the processing time difference, but I cannot explain the difference on memory usage. Is it something intrinsic to the methods, or is it my computer configuration? Regardless the matrix size (at least for matrices larger than 1000x1000), these plateau are the same.
Thanks you for your help.

Output:
It is because you are doing matrix multiplication in pytorch but element-wise multiplication in tensorflow. To do matrix multiplication in TF, use tf.matmul or simply:
for i in range(10000):
  C = A @ B

That does the same for both TF and torch. You also have to call torch.cuda.synchronize() inside the time measurement and move torch.cuda.empty_cache() outside of the measurement for the sake of fairness.
The expected results will be tensorflow's eager execution slower than pytorch.
Regarding the memory usage, TF by default claims all GPU memory and using nvidia-smi in linux or similarly task manager in windows, does not reflect the actual memory usage of the operations.


Instruction: Why does the activation function degrade the time performance of the CNN model?

Input:
I'm testing the CNN model time and accuracy performance with some different small changes. In this model some activation layers between convolutions are skipped. So I added them. I noticed that model with added ELU activations has

performance 3.07 ms per image and 92.26% accuracy on test set

And model without them has

performance 3.52 ms per image and 92.34% accuracy on test set

Although the number of model parameters is the same. So my question is why does activations decrease model's time performance so much?
Here for instance part of code:
...
nn.Conv2d(28 * filters_multiplier, 28 * filters_multiplier, kernel_size=(3, 1), padding=(1, 0)),
nn.Conv2d(28 * filters_multiplier, 28 * filters_multiplier, kernel_size=(1, 3), padding=(0, 1)),

# nn.ELU(),

nn.Conv2d(28 * filters_multiplier, 28 * filters_multiplier, kernel_size=(3, 1), padding=(1, 0)),
nn.Conv2d(28 * filters_multiplier, 40 * filters_multiplier, kernel_size=(1, 3), padding=(0, 1)),
...

So code with commented block works faster than not commented.
UPD I just tested with ReLU() function, and it works as fast as first model with no activations. So maybe problem just in ELU()?

Output:
Neither ReLU nor ELU have learnable parameters, but they still require compute to execute.
ELU executes the exponential function on all x >= 0. This is computational expensive and so your network is slower.
ReLU is a computational cheap because of the ease of the operation x[x < 0] = 0 and so you don't really see a spike in time. This is one of the reasons why ReLU is a common choice as activation function.


Instruction: Syntax Error when calling the name of a model's layer in captum

Input:
I'm trying to use the gradCAM feature of captum for PyTorch. Previously, I asked the question of how to find the name of layers in pyTorch (which is done using model.named_modules()). However, since getting the names of the modules (my model name is 'model') I have tried to use it with LayerGradCam from captum and am receiving a syntax error - it seems to always happen on the 'number' within the model name.
I import the function with:
from captum.attr import LayerGradCam

I'm a bit of a Python newbie, so I've tried calling both:
layer_gc = LayerGradCam(segmentation_wrapper, model.dl.backbone.layer4.2.conv3)

and:
layer_gc = captum.attr.LayerGradCam(segmentation_wrapper, model.dl.backbone.layer4.2.conv3)

The error message I get is:
  File "gradCAM.py", line 120
    layer_gc = LayerGradCam(segmentation_wrapper, model.dl.backbone.layer4.2.conv3)
                                                                           ^
SyntaxError: invalid syntax

This is really stumping me, so any help is appreciated! Thanks in advance :)

Output:
Array or list indexing is done using [] syntax, not ..
model.dl.backbone.layer4[2]conv3


Instruction: Gradients of loss with respect to random parameters are the same in pytorch

Input:
In the simple code below, I perform a simple linear operation on an input tensor of ones and compute its binary cross-entropy loss considering a vector of zeros as the expected output.
When computing the gradient of the loss with respect to w, the rows are the same and equal to the gradient with respect to b. This is counter-intuitive since w and b have random values. What is the reason?
n_input, n_output = 5, 3
x = torch.ones(n_input)
y = torch.zeros(n_output) # expected output
w = torch.randn(n_input, n_output, requires_grad=True)
b = torch.randn(n_output, requires_grad=True)
z = torch.matmul(x,w) + b
loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)

loss.backward()
print(w.grad)
print(b.grad)

Output:
tensor([[0.2179, 0.4337, 0.1959],
        [0.2179, 0.4337, 0.1959],
        [0.2179, 0.4337, 0.1959],
        [0.2179, 0.4337, 0.1959],
        [0.2179, 0.4337, 0.1959]])
tensor([0.2179, 0.4337, 0.1959])

Output:
It's because Your input is symmetric.
Imagine the issue from the point of view of a perceptron (You have 3 of them in Your setup):
each input is 1.0 so the weights of a specific neuron don't matter (it is not important from which input You will take as there is 1.0 everywhere).
If You diversify the input, everything works just fine:
    n_input, n_output = 5, 3
    x = torch.randn(n_input)
    y = torch.ones(n_output)/2.  # expected output
    w = torch.randn(n_input, n_output, requires_grad=True)
    b = torch.randn(n_output, requires_grad=True)
    z = torch.matmul(x, w) + b

    loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)
    loss.backward()
    print(w.grad)
    print(b.grad)

    tensor([[-0.1939,  0.1657, -0.2501],
        [ 0.0561, -0.0480,  0.0724],
        [-0.3162,  0.2703, -0.4079],
        [ 0.0947, -0.0809,  0.1221],
        [-0.0140,  0.0120, -0.0181]])
    tensor([-0.1263,  0.1080, -0.1630])


Instruction: Pytorch - element 0 of tensors does not require grad and does not have a grad_fn - Adding and Multiplying matrices as NN step parameters

Input:
I am a newbie in PyTorch.
I have implemented a custom model (based on a research paper), I get this error when trying to train it.
element 0 of tensors does not require grad and does not have a grad_fn
Here is my code for model:
class Classification(tnn.Module):
    def __init__(self, n_classes=7):
        super(Classification, self).__init__()
        self.feature_extractor1 = VGG16FeatureExtactor() # tnn.Module
        self.feature_extractor2 = VGG16FeatureExtactor() # tnn.Module
        self.conv = conv_layer_relu(chann_in=512, chann_out=512, k_size=1, p_size=0) # tnn.Sequential
        self.attn1 = AttentionBlock1() # tnn.Module
        self.attn2 = AttentionBlock2() # tnn.Module

        # FC layers
        self.linear1 = vggFCLayer(512, 256) # tnn.Sequential
        self.linear2 = vggFCLayer(256, 128) # tnn.Sequential

        # Final layer
        self.final = tnn.Linear(128, n_classes)

    def forward(self, x, x_lbp):
        features1 = self.feature_extractor1(x)
        features2 = self.feature_extractor2(x_lbp)
        f3 = torch.add(features1, features2)
        # Apply attention block1 to features1
        attn1 = self.attn1(features1)
        # Create mask using Attention Block 2
        attn2 = self.attn2(features2)
        mask = attn1 * attn2
        add = f3 + mask

        out = self.conv(add)
        out = out.view(out.size(0), -1)
        out = self.linear1(out)
        out = self.linear2(out)
        out = self.final(out)
        return out

Here is my code for training:
criterion = nn.CrossEntropyLoss(weight=weights)
optimizer = optim.SGD(model.parameters(), lr=lr,momentum=0.9, nesterov=True)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')

    def train(self, epoch, trainloader):
        self.model.train()
        for batch_idx, (inputs, targets) in enumerate(trainloader):
            lbp = lbp_transform(inputs)
            self.optimizer.zero_grad()
            with torch.no_grad():
                outputs = self.model(inputs, lbp)
            loss = self.criterion(outputs, targets)
            loss.backward()
            self.scheduler.step(loss)
            utils.clip_gradient(self.optimizer, 0.1)
            self.optimizer.step()

Edited
Here is Full error stacktrace:
RuntimeError                              Traceback (most recent call last)
<ipython-input-17-b4e6536e5301> in <module>()
      1 modelTraining = train_vggwithattention.ModelTraining(model = model , criterion=criterion,optimizer=optimizer, scheduler=scheduler, use_cuda=True)
      2 for epoch in range(start_epoch, num_epochs):
----> 3     modelTraining.train(epoch=epoch, trainloader=trainloader)
      4     # PublicTest(epoch)
      5     # writer.flush()

2 frames
/content/train_vggwithattention.py in train(self, epoch, trainloader)
     45             loss = self.criterion(outputs, targets)
     46             # loss.requires_grad = True
---> 47             loss.backward()
     48             for name, param in self.model.named_parameters():
     49                 print(name, param.grad)

/usr/local/lib/python3.7/dist-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    253                 create_graph=create_graph,
    254                 inputs=inputs)
--> 255         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    256
    257     def register_hook(self, hook):

/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    147     Variable._execution_engine.run_backward(
    148         tensors, grad_tensors_, retain_graph, create_graph, inputs,
--> 149         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag
    150
    151

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

Any help is appreciated. Thanks!

Output:
You are inferring the outputs using the torch.no_grad() context manager, this means the activations of the layers won't be saved and backpropagation won't be possible.
Therefore, you must replace the following lines in your train function:
        with torch.no_grad():
            outputs = self.model(inputs, lbp)

with simply:
        outputs = self.model(inputs, lbp)


Instruction: How to move feature maps across images using slicing?

Input:
I am trying to implement the online algorithm of this paper, which is on video classification. This work moves 1/8 of channel feature maps from each image, into the next image, after each convolution operation. The image of the operation has been attached here -

While trying to implement the same, I have succeeded in extracting out the first 1/8 channel feature maps, but I don't know how to add them to the succeeding image. My code has been attached below -
import cv2
import gym
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
import torch.autograd as autograd
import torch.nn.functional as F

N = 1 # Batch Size
T = 5 # Time Steps. This means that there are 5 frames in the video
C = 3 # RGB Channels
H = 144 # Height
W = 144 # Width

foo = torch.randn(N*T, C, H, W)

print("Shape of foo = ", foo.shape)
#torch.Size([5, 3, 144, 144])

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 8, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        print("Shape of x = ", x.shape)
        # torch.Size([5, 8, 140, 140])
        shape_extract = x[:, :1,:,:]
        print("Shape of extract = ", shape_extract.shape)
        # torch.Size([5, 1, 140, 140])
        # 1/8 of the channels have been extracted out from above. But how do I transfer these channel features to the next image?

        return x

net = Net()
output = net(foo)

Output:
Since your whole sequence is inside the batch, you can shift the layers using torch.roll the elements on the first axis.
>>> rolled = x.roll(shifts=1, dims=1)

Going from this layer layout on axis=1:
[x_0, x_1, x_2, x_3, ..., x_7]

to this one:
[x_7, x_0, x_1, x_2, ..., x_6]

Then replacing the first element by x_0:
>>> rolled[:, 0] = x[:, 0]

Resulting in this layout:
[x_0, x_0, x_1, x_2, ..., x_6]

Then you can input tensor rolled into the next layer.

You can implement a custom layer to wrap this logic:
class ShiftLayer(nn.Module):
    def forward(self, x):
        out = x.roll(1, 1)
        out[:, 0] = x[:, 0]
        return out

Then use it inside your model:
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        ...
        self.shift = ShiftLayer()

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = self.shift(x)
        x = F.relu(self.conv2(x))
        return x


Instruction: How to fix AttributeError: 'tuple' object has no attribute 'to'?

Input:
Today i faced with problem
AttributeError: 'tuple' object has no attribute 'to'
I read a data from csv file with 2 columns: Image (where the file path is) and finding (where the photo's label is)
Model:
model = models.resnet18(pretrained=False)
model.fc = nn.Sequential(nn.Linear(model.fc.in_features, 256),
                         nn.ReLU(),
                         nn.Dropout(p=0.3),
                         nn.Linear(256, 100),
                         nn.ReLU(),
                         nn.Dropout(p=0.4),
                         nn.Linear(100,9))
# model.load_state_dict(torch.load('model.pth'))

for name, param in model.named_parameters():
    if("bn" not in name):
        param.requires_grad = False

Transforms:
transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.5457, 0.5457, 0.5457], std=[0.2342, 0.2342, 0.2342])
])

Dataset class:
class Col(Dataset):
    def __init__(self, csv, main_dir, transform):
        self.df = pd.read_csv(csv)
        self.main_dir = main_dir
        self.transform = transform

    def __len__(self):
        return self.df.shape[0]

    def __getitem__(self, idx):
        image = transform(Image.open(self.df.Image[idx]).convert("RGB"))
        label = self.df.Finding[idx]
        return image, label

Prepairing Data:
data = Col(main_dir=root_dir,csv=csv_file, transform=transform)
dataset = torch.utils.data.DataLoader(data, batch_size=130)
train_set, validate_set= torch.utils.data.random_split(dataset, [round(len(dataset)*0.7), (len(dataset) - round(len(dataset)*0.7))])

Train func:
def train(model, optimizer, loss_fn, train_set, validate_set, epochs=20, device="cpu"):
    for epoch in range(1, epochs+1):
        training_loss = 0.0
        valid_loss = 0.0
        model.train()
        for batch in train_set:
            optimizer.zero_grad()
            inputs, labels = batch
            inputs = inputs.to(device)
            labels = labels.to(device)
            output = model(inputs)
            loss = loss_fn(output, labels)
            loss.backward()
            optimizer.step()
            training_loss += loss.data.item() * inputs.size(0)
        training_loss /= len(train_set.dataset)

        model.eval()
        num_correct = 0
        num_examples = 0
        for batch in validate_set:
            inputs, labels = batch
            inputs = inputs.to(device)
            output = model(inputs)
            labels = labels.to(device)
            loss = loss_fn(output, labels)
            valid_loss += loss.data.item() * inputs.size(0)
            correct = torch.eq(torch.max(F.softmax(output, dim=1), dim=1)[1], targets)
            num_correct += torch.sum(correct).item()
            num_examples += correct.shape[0]
        valid_loss /= len(validate_set.dataset)

        print('Epoch: {}, Training Loss: {:.2f}, Validation Loss: {:.2f}, accuracy = {:.2f}'.format(epoch, training_loss,
        valid_loss, num_correct / num_examples))

Optimazer:
optimizer = optim.Adam(model.parameters(), lr=0.0001)

After calling train func
train(model, optimizer,torch.nn.CrossEntropyLoss(), train_set.dataset, validate_set.dataset, epochs=100, device=device)

I've got this error:
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
/tmp/ipykernel_9156/634509595.py in <module>
----> 1 train(model, optimizer,torch.nn.CrossEntropyLoss(), train_set.dataset, validate_set.dataset, epochs=100, device=device)

/tmp/ipykernel_9156/2858123881.py in train(model, optimizer, loss_fn, train_set, validate_set, epochs, device)
      8             inputs, labels = batch
      9             inputs = inputs.to(device)
---> 10             labels = labels.to(device)
     11             output = model(inputs)
     12             loss = loss_fn(output, labels)

AttributeError: 'tuple' object has no attribute 'to'

If i try to call this
for batch in train_set.dataset:
    inputs, labels = batch
    print(labels)

then it will display many large tuples with labels.(like this)
('polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps', 'polyps')

How can i fix this? Help me please

Output:
You need to encode your labels into integers (or one-hot encodings depending on your loss_fn).
You should have classes, a list of 9 class names. If not you can get it with self.df.Finding.unique().
You can then invert this list into a dictionnary, essentially a label-name -> label-id mapping:
self.encode = {k: i for i, k in enumerate(classes)}

Then in your dataset __getitem__ definition convert the label names to label ids with encode:
def __getitem__(self, idx):
    image = transform(Image.open(self.df.Image[idx]).convert("RGB"))
    label = self.encode[self.df.Finding[idx]]
    return image, label


Instruction: Count Unique elements in pytorch Tensor

Input:
Suppose I have the following tensor: y = torch.randint(0, 3, (10,)). How would you go about counting the 0's 1's and 2's in there?
The only way I can think of is by using collections.Counter(y) but was wondering if there was a more "pytorch" way of doing this. A use case for example would be when building the confusion matrix for predictions.

Output:
You can use torch.unique with the return_counts option:
>>> x = torch.randint(0, 3, (10,))
tensor([1, 1, 0, 2, 1, 0, 1, 1, 2, 1])

>>> x.unique(return_counts=True)
(tensor([0, 1, 2]), tensor([2, 6, 2]))


Instruction: Pytorch - How differentiate wrt two parameters

Input:
I am interested in combined derivatives using Pytorch:
In the implemented code below, I have tried, but the code compute two partial derivative (e.g. it computed firstly d'f/d'x and secondly d'f/d'y). Is it possible modify the code in some way that we can compute this derivative with respect two parameters?
import torch
def function(x,y):
    f = x**3+y**3
    return f

a =  torch.tensor([4., 5., 6.], requires_grad=True)
b =  torch.tensor([1., 2., 6.], requires_grad=True)
derivative = torch.autograd.functional.jacobian(function, (a,b))
print(derivative)

Thanks in advance!

Output:
You can use torch.autograd.functional.hessian to get the combined derivatives.
>>> f = lambda x, y: (x**3 + y**3).mean()
>>> H = A.hessian(f, (a, b))

Since you have two inputs, the result will be a tuple containing 2 tuples.
More precisely, you will have

H[0][0] the 2nd derivative w.r.t x: d²z_i/dx_j*dx_j

H[0][1] the combined derivative w.r.t x and y: d²z_i/dx_j*dy_j

H[0][1] the combined derivative w.r.t y and x: d²z_i/dy_j*dx_j

H[1][1] the 2nd derivative w.r.t y: d²z_i/dy_j*dy_j

>>> H
((tensor([[ 8.,  0.,  0.],
          [ 0., 10.,  0.],
          [ 0.,  0., 12.]],
  tensor([[ 0.,  0.,  0.],
          [ 0.,  0.,  0.],
          [ 0.,  0.,  0.]]),
 (tensor([[ 0.,  0.,  0.],
          [ 0.,  0.,  0.],
          [ 0.,  0.,  0.]]))
  tensor([[ 2.,  0.,  0.],
          [ 0.,  4.,  0.],
          [ 0.,  0., 12.]])

Indeed if you look at the combined derivative: d²(x³+y³)/dxdy = d(3x²)/dy = 0, hence H[0][1] and H[1][0] are zero matrices.
On the other hand we have d²x³/d²x = 6x, since the f is averaging the values, it gives 6x/3 = 2x. Similarly, you get d²x³/d²y = 6y.
As a result, you find that H[0][0] = diag(2a) and H[1][1] = diag(2b).


Instruction: Model takes twice the memory footprint with distributed data parallel

Input:
I have a model that trains just fine on a single GPU. But I'm getting CUDA memory errors when I switch to Pytorch distributed data parallel (DDP). Specifically, the DDP model takes up twice the memory footprint compared to the model with no parallelism. Here is a minimal reproducible example:
import os
from torch.nn.parallel import DistributedDataParallel as DDP
import torch.distributed as dist
import torch.multiprocessing as mp
import torch

def train(rank, gpu_list, train_distributed):

    device_id = gpu_list[rank]

    model = torch.nn.Linear(1000, 1000)
    print(device_id, torch.cuda.memory_allocated(device_id))
    model.to(device_id)
    print(device_id, torch.cuda.memory_allocated(device_id))

    print(device_id, torch.cuda.memory_allocated(device_id))
    if train_distributed:
        # convert model to DDP
        dist.init_process_group("gloo", rank=rank, world_size=len(gpu_list))
        model = DDP(model, device_ids=[device_id], find_unused_parameters=False)
    print(device_id, torch.cuda.memory_allocated(device_id))

def train_distributed():
    gpu_list = [torch.device(i) for i in [5, 6]]
    os.environ['MASTER_ADDR'] = '127.0.01'
    os.environ['MASTER_PORT'] = '7676'
    mp.spawn(train, args=(gpu_list, True), nprocs=len(gpu_list), join=True)

if __name__ == '__main__':
    # First test one GPU
    train(0, [torch.device(5)], False)

    # Then test multiple GPUs
    train_distributed()

Output - note that the GPU usage doubles on both devices when switching to DDP:
cuda:5 0
cuda:5 4004352
cuda:5 4004352
cuda:5 4004352
cuda:5 0
cuda:6 0
cuda:5 4004352
cuda:5 4004352
cuda:6 4004352
cuda:6 4004352
cuda:5 8008704
cuda:6 8008704

Why does the model take up twice the space in DDP? Is it intended behavior? Is there a way to avoid this extra memory usage?

Output:
I'm adding here the solution of @ptrblck written in the PyTorch discussion forum.
Here're two quotes.
The statement:

[...] the allocated memory get doubled when torch.distributed.Reducer is instantiated in the constructor of DistributedDataParallel

And the answer:

[...] the Reducer will create gradient buckets for each parameter, so that the memory usage after wrapping the model into DDP will be 2x model_parameter_size. Note that the parameter size of a model is often much smaller than the activation size so that this memory increase might or might not be significant

So, from here we can see the reason why the memory footprint sometimes doubles.


Instruction: Transformers Longformer IndexError: index out of range in self

Input:
From Transformers library I use LongformerModel, LongformerTokenizerFast, LongformerConfig (all of them use from_pretrained("allenai/longformer-base-4096")).
When I do
longformer(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)

I get such an error:
~/tfproject/tfenv/lib/python3.7/site-packages/transformers/modeling_longformer.py in forward(self, input_ids, token_type_ids, position_ids, inputs_embeds)
    177         if inputs_embeds is None:
    178             inputs_embeds = self.word_embeddings(input_ids)
--> 179         position_embeddings = self.position_embeddings(position_ids)
    180         token_type_embeddings = self.token_type_embeddings(token_type_ids)
    181

~/tfproject/tfenv/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

~/tfproject/tfenv/lib/python3.7/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    112         return F.embedding(
    113             input, self.weight, self.padding_idx, self.max_norm,
--> 114             self.norm_type, self.scale_grad_by_freq, self.sparse)
    115
    116     def extra_repr(self):

~/tfproject/tfenv/lib/python3.7/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1722         # remove once script supports set_grad_enabled
   1723         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1724     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1725
   1726

IndexError: index out of range in self

Online I found that this might mean that my input to the model has more tokens than the model's max input size.
But I have checked and all inputs have exactly 4098 tokens (which is the model max length of input size) (padding has been applied). Tokenizer has the same vocab size as the model.
I have no idea what's wrong.

Output:
I have managed to fix this by reindexing my position_ids.
When PyTorch was creating that tensor, for some reason some value in position_ids was bigger than 4098.
I used:
position_ids = torch.stack([torch.arange(config.max_position_embeddings) for a in range(val_dataloader.batch_size)]).to(device)

to create position_ids for the entire batch.
Bear in mind that it might not be the best solution. The problem might need some more debugging. But for a quick fix, it works.


Instruction: does is affect NN training accuracy if the color format of images is BGR not RGB?

Input: I'm training Neural Network with ImageNet dataset and I noticed that images are in BGR color format when I read them using OpenCV function cv2.imread(), so does is affect training accuracy?, if yes then how can I change it to RGB in pytorch?

Output:
It will not affect your NN's accuracy, in general. However, if you are using a pre-trained CNN, then it likely expects RGB images as input, and will not do as well on BGR images initially and will have to re-learn its weights for BGR.
You can convert BGR to RGB using cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB).
You can also consider the following alternatives for reading images:

torchvision.io.read_image(path) (https://pytorch.org/vision/stable/io.html#image)
torch.from_numpy(np.array(PIL.Image.open(path)))
torchvision.transforms.functional.pil_to_tensor(PIL.Image.open(path)) (https://pytorch.org/vision/stable/transforms.html#torchvision.transforms.functional.pil_to_tensor)