Instruction: Using hook method in a data parallelism approach in PyTorch

Input:
A pretrained model, as my encoder, has several layers as follows and I want to extract some features from some layers of that :
TimeSformer(
  (model): VisionTransformer(
(dropout): Dropout(p=0.0, inplace=False)
(patch_embed): PatchEmbed(
  (proj): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))
)
(pos_drop): Dropout(p=0.0, inplace=False)
(time_drop): Dropout(p=0.0, inplace=False)
(blocks): ModuleList(  #************
  (0): Block(
    (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (attn): Attention(
      (qkv): Linear(in_features=768, out_features=2304, bias=True)
      (proj): Linear(in_features=768, out_features=768, bias=True)
      (proj_drop): Dropout(p=0.0, inplace=False)
      (attn_drop): Dropout(p=0.0, inplace=False)
    )
    (temporal_norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (temporal_attn): Attention(
      (qkv): Linear(in_features=768, out_features=2304, bias=True)
      (proj): Linear(in_features=768, out_features=768, bias=True)
      (proj_drop): Dropout(p=0.0, inplace=False)
      (attn_drop): Dropout(p=0.0, inplace=False)
    )
    (temporal_fc): Linear(in_features=768, out_features=768, bias=True)
    (drop_path): Identity()
    (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (mlp): Mlp(
      (fc1): Linear(in_features=768, out_features=3072, bias=True)
      (act): GELU()
      (fc2): Linear(in_features=3072, out_features=768, bias=True)
      (drop): Dropout(p=0.0, inplace=False)
    )
  )
  (1): Block(
    (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (attn): Attention(
      (qkv): Linear(in_features=768, out_features=2304, bias=True)
      (proj): Linear(in_features=768, out_features=768, bias=True)
      (proj_drop): Dropout(p=0.0, inplace=False)
      (attn_drop): Dropout(p=0.0, inplace=False)
    )
    (temporal_norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (temporal_attn): Attention(
      (qkv): Linear(in_features=768, out_features=2304, bias=True)
      (proj): Linear(in_features=768, out_features=768, bias=True)
      (proj_drop): Dropout(p=0.0, inplace=False)
      (attn_drop): Dropout(p=0.0, inplace=False)
    )
    (temporal_fc): Linear(in_features=768, out_features=768, bias=True)
    (drop_path): DropPath()
    (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
    (mlp): Mlp(
      (fc1): Linear(in_features=768, out_features=3072, bias=True)
      (act): GELU()
      (fc2): Linear(in_features=3072, out_features=768, bias=True)
      (drop): Dropout(p=0.0, inplace=False)
    )
  )
.
.
.
.
.
.

I need to extract features of specific layers of the above mentioned model and I am using hook method as follows:
import torch.nn as nn
class my_class(nn.Module):
    def __init__(self, pretrained=False):
        super(my_class, self).__init__()

        self.featureExtractor =TimeSformer(img_size=224, num_classes=400, num_frames=8, attention_type='divided_space_time',
                                           pretrained_model='/home/TimeSformer_divST_16x16_448_K400.pyth')

        self.featureExtractor=nn.DataParallel(self.featureExtractor)

        self.list = list(self.featureExtractor.children())

        self.activation = {}
        def get_activation(name):
            def hook(model, input, output):
              self.activation[name] = output.detach()
            return hook

        self.featureExtractor.model.blocks[4].register_forward_hook(get_activation('block4'))  # line 30 in the error
        self.featureExtractor.model.blocks[8].register_forward_hook(get_activation('block8'))
        self.featureExtractor.model.blocks[4].temporal_attn.register_forward_hook(get_activation('block4.temporal_attn'))
        self.featureExtractor.model.blocks[11].register_forward_hook(get_activation('block11'))

    def forward(self, x, out_consp = False):

       b = self.featureExtractor(x)

       block4_output_Temporal_att = self.activation['block4.temporal_attn']
       block4_output = self.activation['block4']

       block8_output = self.activation['block8']
       block11_output = self.activation['block11']
.
.
.
.

The problem is I face the following error:

File "/home/TimeSformer/models/timesformer.py", line 30, in init
self.featureExtractor.model.blocks[4].register_forward_hook(get_activation('block4'))

File "/home/miniconda3/envs/pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1185, in getattr
raise AttributeError("'{}' object has no attribute '{}'".format(
AttributeError: 'DataParallel' object has no attribute 'model'

How can I solve the problem

Output:
You need to access the module attribute, not model. Refer to line 140 of this file.
In other words, you should access your blocks via self.featureExtractor.module.blocks.


Instruction: Pytorch progress bar disappear on vscode jupyter

Input: I have problem when training Pytorch model, the progress bar of is disappeared by no reason today. It still work properly the days before. I'm using jupyter through vs code, connect to the kernel that run on the Ubuntu subsystem. How can I show the progress bar as normal

Output:
I had this issue and it seems to come from a problem with tqdm for new versions of ipywidget (see https://github.com/microsoft/vscode-jupyter/issues/8552).
As mentioned in the link, I solved it by downgrading ipywidgets:
pip install ipywidgets==7.7.2


Instruction: The size of Logits of Roberta model is weird

Input:
My input size is [8,22]. A batch with 8 tokenized sentences with a length of 22.
I dont want to use the default classifier.
model = RobertaForSequenceClassification.from_pretrained("xlm-roberta-large")

model.classifier=nn.Identity()

After model(batch)
The size of result is torch.Size([8, 22, 1024]). I have no idea why. Should it be [8,1024]?

Output:
The model.classifier object you have replaced used to be an instance of a RobertaClassificationHead. If you take a look at its source code[1], the layer is hard-coded into indexing the first item of the second dimension of its input, which is supposed to be the [CLS] token.
By replacing it with an Identity you miss out on the indexing operation, hence your output shape.
Long story short, don't assume functionality you haven't verified when it comes to non-own code, huggingface in particular (lots of ad-hoc classes and spaghetti interfaces, least as far as I'm concerned).

[1] source


Instruction: Output of layer in CNN with equal dim of feature

Input:
I read a bit about autoencoders and read that encoder transform data from $\mathbb{R}^{N \times D} \rightarrow \mathbb{R}^{N\times d}$$ space.
I wrote CNN autoencoder with for example one layer like this:
  self.conv1 = nn.Conv2d(1, 1, 5)

but when I apply x = F.relu(self.conv1(x)) of course I get an image with smaller dimensions (if I had the dimension torch.Size([1, 400, 1024]) at the beginning)
after the operation I have torch.Size([1, 396, 1020]).
It is clear that convolution reduces the dimension of the image (that's all clear :) ), but I'm interested in whether there is a way to keep the number of features and then fill the others with zeros, and if that would be a step in the right direction, the question arises as to which features to fill with zeros .
P.S. I use only one channel because I want the final result to be in a 2d matrix, and my images are gray scale anyway.

Output:
I think you need to add a padding='same' option
self.conv1 = nn.Conv2d(1, 1, 5, padding='same')

This is the usual way of ensuring output dimensions don't get shrunk
when applying convolutional layers. It pads the original image before
applying the convolution, by default with zeros, rather than padding the result.
(If this were Tensorflow I would be certain, but it looks like torch works
the same way. https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html)


Instruction: why do object detection methods have an output value for every class

Input:
Most recent object detection methods rely on a convolutional neural network. They create a feature map by running input data through a feature extraction step. They then add more convolutional layers to output a set of values like so (this set is from YOLO, but other architectures like SSD differ slightly):
pobj: probability of being an object
c1, c2 ... cn: indicating which class the object belongs to
x, y, w, h: bounding box of the object
However, one particular box cannot be multiple objects. As in, wouldn't having a high value for, say, c1 mean that the values for all the others c2 ... cn would be low? So why use different values for c1, c2 ... cn? Couldn't they all be represented by a single value, say 0-1, where each object has a certain range within the 0-1, say 0-0.2 is c1, 0.2-0.4 is c2 and so on...
This would reduce the dimension of the output from NxNx(5+C) (5 for the probability and bounding box, +C one for each class) to NxNx(5+1) (5 same as before and 1 for the class)
Thank you

Output:
Short answer, NO! That is almost certainly not an acceptable solution. It sounds like your core question is:  Why is a a single value in the range [0,1] not a sufficient, compact output for object classification? As a clarification, I'd say this doesn't really have to do with single-shot detectors; the outputs from 2-stage detectors and most all classification networks follows this same 1D embedding structure. As a secondary clarification, I'd say that many 1-stage networks also don't output   pobj  in their original implementations (YOLO is the main one that does but Retinanet and I believe SSD does not).
An object's class is a categorical attribute. Assumed within a standard classification problem is that the set of possible classes is flat (i.e. no class is a subclass of any other), mutually exclusive (each example falls into only a single class), and unrelated (not quite the right term here but essentially no class is any more or less related to any other class).
This assumed attribute structure is well represented by an orthonormal encoding vector of the same length as the set of possible attributes. A vector [1,0,0,0] is no more similar to [0,1,0,0] than it is to [0,0,0,1] in this space.
(As an aside, a separate branch of ML problems called multilabel classification removes the mutual exclusivity constrain (so [0,1,1,0] and [0,1,1,1] would both be valid label predictions. In this space class or label combinations COULD be construed as more or less related since they share constituent labels or "basis vectors" in the orthonormal categorical attribute space. But enough digression..)
A single, continuous variable output for class destroys the assumption that all classes are unrelated. In fact, it assumes that the relation between any two classes is exact and quantifiable! What an assumption! Consider attempting to arrange the classes of, let's say, the ImageNet classification task, along a single dimension. Bus and car should be close, no? Let's say 0.1 and 0.2, respectively in our 1D embedding range of [0,1]. Zebra must be far away from them, maybe 0.8. But should be close to zebra fish (0.82)? Is a striped shirt closer to a zebra or a bus? Is the moon more similar to a bicycle or a trumpet? And is a zebra really 5 times more similar to a zebra fish than a bus is to a car? The exercise is immediately, patently absurd. A 1D embedding space for object class is not sufficiently rich to capture the differences between object classes.
Why can't we just place object classes randomly in the continuous range [0,1]? In a theoretical sense nothing is stopping you, but the gradient of the network would become horrendously, unmanageably non-convex and conventional approaches to training the network would fail. Not to mention the network architecture would have to encode extremely non-linear activation functions to predict the extremely hard boundaries between neighboring classes in the 1D space, resulting in a very brittle and non-generalizable model.
From here, the nuanced reader might suggest that in fact, some classes ARE related to one another (i.e. the unrelated assumption of the standard classification problem is not really correct). Bus and car are certainly more related than bus and trumpet, no? Without devolving into a critique on the limited usefulness of strict ontological categorization of the world, I'll simply suggest that in many cases there is an information embedding that strikes a middle ground. A vast field of work has been devoted to finding embedding spaces that are compact (relative to the exhaustive enumeration of "everything is its own class of 1") but still meaningful. This is the work of principal component analysis and object appearance embedding in deep learning.
Depending on the particular problem, you may be able to take advantage of a more nuanced embedding space better suited towards the final task you hope to accomplish. But in general, canonical deep learning tasks such as classification / detection ignore this nuance in the hopes of designing solutions that are "pretty good" generalized over a large range of problem spaces.


Instruction: Drastic difference in accuracy for varying batch sizes?

Input:
When training my CNN image classifier using PyTorch I noticed a ~20+% difference in accuracy when using a batch size of 4 vs 32. What might be causing such drastic differences?
batch_size 4
100%|██████████| 10/10 [02:50<00:00, 17.04s/it, TestAcc=71%, TrainAcc=74%, loss=0.328]

batch_size 32
100%|██████████| 10/10 [02:38<00:00, 15.85s/it, TestAcc=53%, TrainAcc=57%, loss=0.208]

Model:
class Net(nn.Module):
def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(3, 6, 5)
    self.pool = nn.MaxPool2d(2, 2)
    self.conv2 = nn.Conv2d(6, 16, 5)
    self.fc1 = nn.Linear(16 * 5 * 5, 120)
    self.fc2 = nn.Linear(120, 84)
    self.fc3 = nn.Linear(84, num_classes)

def forward(self, x):
    x = self.pool(F.relu(self.conv1(x)))
    x = self.pool(F.relu(self.conv2(x)))
    x = torch.flatten(x, 1)  # flatten all dimensions except batch
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return x

Output:
You can try to adjust your learning rate too.
With a larger batch size you should also use a larger learning rate.
This article has additional explanations for the relation of learning rate and batch size.


Instruction: Pytorch gives error Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu

Input:
I'm beggining my journey to Python 3 / Pytorch, and i'm having the following issue:
I'm trying to run this code on my GPU, but i get the following error :

Expected all tensors to be on the same device, but found at least two
devices, cuda:0 and cpu!

I know it means i'm trying to manipulate 2 tensors that are both on different devices, but i can figure out where in my code I missed to transfer this tensor.
Any help would be appreciated.
Here is the code
import sys
import os
sys.path.append(os.path.abspath("include"))
sys.path.append(os.path.abspath("models"))

from utils import *
from MD5EncryptedDataEncoder import *

import torch
import torch.nn.functional as F
from torch import nn

import numpy as np
import hashlib
import pathlib
from pathlib import Path

# instantiate the model
model = MD5EncryptionEncoder()
model_file = Path(model.m_save_path)
if model_file.is_file():
    model = torch.load(model.m_save_path)
    print("Loaded previously saved model from ["+model.m_save_path+"]")
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
print("Device that will be used : ")
print(device)
model = model.to(device)
print("Model parameter devices :")
print(next(model.parameters()).device)

#define our loss function
loss_function = nn.MSELoss()
loss_function = loss_function.to(device)

#defined our optimizer

#learning_rate = 0.00000005
#optimizer = torch.optim.RMSprop(model.parameters(), lr=learning_rate)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.7,patience=8,cooldown=2,verbose=True)

loss_history = [0.02]
for i in range(50000000):

    #Create our random unencrypted md5 data + convert it to a pytorch tensor
    md5_unencrypted_data_hexadecimal_string = secrets.token_hex(64)
    md5_unencrypted_data_tensor = HexStringToBinaryTensor(md5_unencrypted_data_hexadecimal_string)
    md5_unencrypted_data_binary_blob = HexStringToBinaryBlob(md5_unencrypted_data_hexadecimal_string)

    #Run the MD5 algorithm on our unencrypted md5 data + convert the result to a pytorch tensor
    md5_encrypted_data = hashlib.md5(md5_unencrypted_data_binary_blob)
    md5_encrypted_data_hexadecimal_string = md5_encrypted_data.hexdigest()
    md5_encrypted_data_tensor = HexStringToBinaryTensor(md5_encrypted_data_hexadecimal_string)
    md5_encrypted_data_tensor = md5_encrypted_data_tensor.to(device)
    print(md5_encrypted_data_tensor)

    #print("Unencrypted tensor :")
    #print(md5_unencrypted_data_tensor)

    #print("Encrypted tensor :")
    #print(md5_encrypted_data_tensor)

    #Run our forward pass
    predictions_tensor = model(md5_encrypted_data_tensor)

    #print("Prediction : ",predictions_tensor)

    #Run our loss function
    loss = loss_function(predictions_tensor, md5_encrypted_data_tensor)

    #Compute the gradient (this does NOT update the weight : only compute the error gradient)
    loss.backward()

    if i % 100 == 0:
        #Update the weights
        optimizer.step()

        #We are not using batch update, so restart the gradients
        optimizer.zero_grad()

        #Call the scheduler that will changer some params if necessary
        scheduler.step(loss)

        print("Loss : ",loss.item())

        if(loss.item() < min(loss_history)):
            print("Saving model to ["+model.m_save_path+"]")
            torch.save(model,model.m_save_path)

        loss_history.append(loss.item())
    if i % 500 == 0:
        print("Prediction : ",predictions_tensor)
        print("Wanted result : ",md5_encrypted_data_tensor)

Here is the class MD5EncryptedDataEncoder code :
import torch
import torch.nn.functional as F
from torch import nn

# define the network class
class MD5EncryptionEncoder(nn.Module):

    m_save_path = "data/MD5EncryptionEncoder.model"

    def __init__(self):
        # call constructor from superclass
        super().__init__()

        self.input_to_hidden_1 = nn.Linear(128, 10240)

        self.hidden_layers = []
        self.hidden_layers.append(nn.Linear(10240, 10240))
        self.hidden_layers.append(nn.Linear(10240, 10240))

        self.last_hidden_to_output = nn.Linear(10240, 128)

        self.hidden_layers_activation_function = torch.nn.LeakyReLU(0.1)

    def forward(self, x):
        # define forward pass.
        # Here, 'x' represent the output of a network defined in __init__

        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))
        x = self.hidden_layers_activation_function(self.hidden_layers[0](x))

        x = self.hidden_layers_activation_function(self.hidden_layers[1](x))
        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x

    def EncryptedTensorToStateTensor(self,x):

        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))
        x = self.hidden_layers_activation_function(self.hidden_layers[0](x))

        return x

    def StateTensorToEncryptedTensor(self,x):

        x = self.hidden_layers_activation_function(self.hidden_layers[1](x))
        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x

Here is the content of utils.py
import torch
import secrets
import binascii

def HexStringToBinaryBlob(p_hex_string):
    if(len(p_hex_string)%2 != 0):
        print("[HexStringToBinaryTensor] Parameter [p_hex_string]'s size if not an even number")
        exit(1)
    binary_blob = binascii.unhexlify(p_hex_string)
    return binary_blob

def HexStringToBinaryTensor(p_hex_string):
    if(len(p_hex_string)%2 != 0):
        print("[HexStringToBinaryTensor] Parameter [p_hex_string]'s size if not an even number")
        exit(1)
    binary_blob = binascii.unhexlify(p_hex_string)
    binary_string = ''.join(format(x, '08b') for x in binary_blob)

    result_tensor_size = int(len(p_hex_string)/2*8)
    result_tensor = torch.randn(result_tensor_size)
    for binary_position in range (len(binary_string)):
        binary_digit = int(binary_string[binary_position])
        result_tensor[binary_position] = binary_digit
    return result_tensor

def Create512BitBinaryTensor():
    #Create a 512 bits (64 BYTES) block of data in hexadecimal
    hexadecimal_blob = secrets.token_hex(64)
    result_tensor = HexStringToBinaryTensor(hexadecimal_blob)
    return result_tensor

Here is the terminal error
Loaded previously saved model from [data/MD5EncryptionEncoder.model]
Device that will be used :
cuda:0
Model parameter devices :
cuda:0
tensor([0., 1., 0., 1., 0., 1., 0., 1., 0., 1., 1., 0., 1., 0., 1., 1., 1., 0.,
        1., 1., 0., 1., 1., 0., 0., 0., 0., 0., 1., 0., 0., 1., 1., 1., 0., 0.,
        1., 1., 1., 0., 1., 0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 0., 0., 1.,
        0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 1., 0., 0., 0., 1.,
        1., 0., 0., 1., 1., 1., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0.,
        1., 1., 1., 0., 0., 1., 1., 0., 1., 1., 1., 1., 1., 1., 1., 0., 0., 1.,
        0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1., 1.,
        1., 0.], device='cuda:0')
Traceback (most recent call last):
  File "TrainEncryptedDataEncoder.py", line 71, in <module>
    predictions_tensor = model(md5_encrypted_data_tensor)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/workspace/Pytorch/models/MD5EncryptedDataEncoder.py", line 45, in forward
    x = self.hidden_layers_activation_function(self.hidden_layers[0](x))
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 114, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument mat in method wrapper_addmv_)

Output:
The issue is the List data type defined in the Model Class.
You have used
self.hidden_layers = []
self.hidden_layers.append(nn.Linear(10240, 10240))
self.hidden_layers.append(nn.Linear(10240, 10240))

Here the defined list will be on cpu only. You need to either put it in a nn.sequential and then call it by index as below:-
class MD5EncryptionEncoder(nn.Module):
    m_save_path = "data/MD5EncryptionEncoder.model"

    def __init__(self):
        # call constructor from superclass
        super().__init__()

        self.input_to_hidden_1 = nn.Linear(128, 10240)
        self.hidden_layers = []
        self.hidden_layers.append(nn.Linear(10240, 10240))
        self.hidden_layers.append(nn.Linear(10240, 10240))
        self.hidden = nn.Sequential(*self.hidden_layers)
        self.last_hidden_to_output = nn.Linear(10240, 128)
        self.hidden_layers_activation_function = torch.nn.LeakyReLU(0.1)

    def forward(self, x):
        # define forward pass.
        # Here, 'x' represent the output of a network defined in __init__
        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))
        x = self.hidden_layers_activation_function(self.hidden[0](x))
        x = self.hidden_layers_activation_function(self.hidden[1](x))
        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x

    def EncryptedTensorToStateTensor(self, x):
        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))
        x = self.hidden_layers_activation_function(self.hidden_layers[0](x))

        return x

    def StateTensorToEncryptedTensor(self, x):
        x = self.hidden_layers_activation_function(self.hidden_layers[1](x))
        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x

or just ditch putting the linear layers in list and just define the layers outside.
class MD5EncryptionEncoder(nn.Module):
    m_save_path = "data/MD5EncryptionEncoder.model"

    def __init__(self):
        # call constructor from superclass
        super().__init__()

        self.input_to_hidden_1 = nn.Linear(128, 10240)

        self.hidden_layers = []
        self.hidden_layer1 = nn.Linear(10240, 10240)
        self.hidden_layer2 = nn.Linear(10240, 10240)
        self.last_hidden_to_output = nn.Linear(10240, 128)

        self.hidden_layers_activation_function = torch.nn.LeakyReLU(0.1)

    def forward(self, x):
        # define forward pass.
        # Here, 'x' represent the output of a network defined in __init__

        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))

        x = self.hidden_layer1(x)
        x = self.hidden_layer2(x)

        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x

    def EncryptedTensorToStateTensor(self, x):
        x = self.hidden_layers_activation_function(self.input_to_hidden_1(x))
        x = self.hidden_layers_activation_function(self.hidden_layers[0](x))

        return x

    def StateTensorToEncryptedTensor(self, x):
        x = self.hidden_layers_activation_function(self.hidden_layers[1](x))
        x = torch.sigmoid(self.last_hidden_to_output(x))

        return x


Instruction: torch.cuda.device_count() returns 2, but torch.load(model_path, map_location='cuda:1') throws an error

Input:
I have two GPUs and when I run
import torch
print('count: ', torch.cuda.device_count())  # prints count: 2

However, my model throws an error
RuntimeError: Attempting to deserialize object on CUDA device 2 but torch.cuda.device_count() is 1

on the line
torch.load(model_path, map_location='cuda:1')

What could cause it and how to fix it?
This issue is somehow linked to my Flask, because the training itself works with torch.load(model_path, map_location='cuda:1')

Output:
This is a known Flask-CUDA issue. Please run Flask with it with
print('count: ', torch.cuda.device_count()) and check if you see
count: 2
reloading
count: 1

If so, add app.run(... , use_reloader=False)


Instruction: Understanding vocab.txt of vinai/bertweet-base

Input:
While looking at vocab.txt here, I was left wondering why the vocabulary for Bertweet is not continuous.
For instance, see the below sample:
...
dice 63328
)@@ 63327
struggled 63326
wraps 63324
Investors 63312
#summer@@ 63305
...

As you can see, after 63305, we have 63312, followed by 63324... what about the numbers in between?

Also, it feels a bit strange why vocabulary starts at around 3800.

르@@ 3800
utory 3798
...

Any explanations will be really appreciated.

Output:
Each number denotes the frequency count that the corresponding word appears in the pre-training corpus.

Only top 64k words are included in the vocab.


Instruction: Sample rows from tensor in every 3 rows in Python

Input:
How to sample the tensor in Python?
I want to sample a frame every 3 frames from videos, and the tensor shape will be [color, frames, height, width].
Thus, the sampling tensor shape will be [color, frames / 3, height, width]
Assume there is a tensor.size([3,300,10,10]).
After sampling rows every 3 rows in the second dimension, the tensor will be tensor.size([3,100,10,10])
Another example,
A tensor = [[1,2,3,4,5,6,7,8,9,10],[1,2,3,4,5,6,7,8,9,10],[1,2,3,4,5,6,7,8,9,10]].
After sampling rows every 3 rows in the second dimension, the tensor will be [[1,4,7,10],[1,4,7,10],[1,4,7,10]]

Output:
Let N be the size of dimension you want to sample and you want to sample every kth row.
You can do (assuming you want to sample from the 1st dimension, and there are 4 dimensions),
new_tensor = tensor[:, torch.arange(0, N, k), : ,: ]
You may skip slicing the last two dimensions and the result won't change.
new_tensor = tensor[:, torch.arange(0, N, k)]
More specifically for the 2D tensor in question, you can use this code.
tensor=torch.tensor([
             [1,2,3,4,5,6,7,8,9,10],
             [1,2,3,4,5,6,7,8,9,10],
             [1,2,3,4,5,6,7,8,9,10]
       ])
new_tensor=tensor[:, torch.arange(0,10,3)]


Instruction: SGD Optimizer Custom Parameters

Input:
I am practicing using Pytorch and trying to implement a simple linear model.
I have initialized some random x and y inputs alongside some random parameters 'a' and 'b'.
import torch
torch.manual_seed(42)
x = torch.rand(100,1)
y = 1 + 2 * x + .1 * torch.rand(100, 1)
a = torch.randn(1, requires_grad=True)
b = torch.randn(1, requires_grad=True)

I would like to use gradient descent on a and b and update them as the model trains. I am using an sklearn linear regression model to compare my results to but they do not seem to be in accordance with each other as I would expect.
Here is the code to create the model
import torch.nn as nn
from torch.optim import SGD

#create simple linear model

class LinearModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(1, 1)

    def forward(self, x):
        return self.linear(x)

#loss function
criterion = torch.nn.MSELoss()

SGDmodel = LinearModel()

I am thinking my problem could possibly be within
sgd = torch.optim.SGD([a, b], lr=0.001, momentum=0.9, weight_decay=0.1)

Can I simply input my a and b parameters into the torch.optim function and have them update as the model is trained? or do I need to embed them into my model.parameters() in a different way somehow?
Here is the rest of my code if needed.
epochs = 1
inputs = x
targets = y

for epoch in range(epochs):

    sgd.zero_grad()
    yhat = SGDmodel(inputs)
    loss = criterion(yhat, targets)
    loss.backward()
    sgd.step()

    print(f'Loss: {loss}')

I am using the following to see what a and b are
print(SGDmodel.linear.weight.grad)
print(SGDmodel.linear.bias.grad)
tensor([[-3.1456]])
tensor([-5.4119])

Any help would be appreciated!

Output:
Can I simply input my a and b parameters into the torch.optim function and have them update as the model is trained? or do I need to embed them into my model.parameters() in a different way somehow?

I think I figured it out!
I added a and b as parameters in my model by implementing torch.nn.Parameter()
I added them into the model, and called self.weight and self.bias.
class LinearModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(1, 1)
        self.weight = torch.nn.Parameter(a) #sets a as the weight in the linear layer
        self.bias = torch.nn.Parameter(b) # sets b as the bias in the linear layer

    def forward(self, x):
        return self.linear(x)

Now when I call my optimizer it will update the custom parameters I defined as a and b
sgd = torch.optim.SGD(SGDmodel.parameters(), lr=0.001, momentum=0.9, weight_decay=0.1)


Instruction: As a non-root user, how to install another version cuda in conda environment in Linux server?

Input:
I am a non-root user in a Linux server, when I input nvidia-smi it shows my cuda is 10.2.
However, my project is asked to
pip install torch==1.10.0+cu111 torchvision==0.11.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
Now, I have create my own environment, for example, cu111, by conda create --name cu111 python=3.10 and conda activate cu111
What should I do step by step, the first step is following pip install torch==1.10.0+cu111 torchvision==0.11.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html ,right?
I read some tutorial, it asked to add path variable in your environment, but I am not sure if it is correct,
export PATH=/home/dj/miniconda3/envs/cu111/lib/:$PATH

Any suggestion is helpful for me!

Output:
I am a non-root user in a Linux server, when I input nvidia-smi it shows my cuda is 10.2.

The "CUDA version" shown by nvidia-smi is the maximum CUDA version supported by the GPU driver installed on the system. That means that no matter what CUDA toolkit or CUDA accelerated framework you choose to install, nothing newer than CUDA 10.2 or anything compiled against anything newer than CUDA 10.2 will actually work on the machine in question. If you install a version of PyTorch compiled against CUDA 11.1, it cannot work on the machine in its current state.
If you are a non-root user, there is nothing you can do to fix this other than ask a system administrator to perform a driver upgrade (if that is possible, i.e. the hardware is actually supported by a more modern driver).


Instruction: why my model does not learn? Same on every epoch (Pytorch)

Input:
from sklearn import datasets
import pandas as pd
import numpy as np
import torch
from torch import nn

#loading the dataset
(data, target) = datasets.load_diabetes(as_frame=True,return_X_y=True) #with the as_frame=True data: pd.DataFrame

# converting data,target to tensors
data = torch.tensor(data.values,dtype=torch.float)
target = torch.tensor(target.values,dtype=torch.float)

#split the data 80% train 20% testing
a = 0.8
train_data , train_target = data[:int(a*len(data))] , data[:int(a*len(data))]
test_data , test_target = data[int(a*len(data)):] , data[int(a*len(data)):]

#constructing the model
# for this dataset dimentionality is 10 so the in_features will be 10
model = nn.Sequential(
    nn.Linear(in_features=10, out_features=128),
    nn.Linear(in_features=128, out_features=128),
    nn.Linear(in_features=128, out_features=1)
)

#loss fn , optimizer
loss_fn = nn.L1Loss() #binary cross entropy
optimizer = torch.optim.SGD(params = model.parameters(),lr=0.001) #stochastic gradient descent

#training loop
epochs = 1000

for epoch in range(epochs):
    #1. make prediction
    model.train()
    train_pred = model(train_data)

    loss = loss_fn(train_pred, train_target)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    model.eval()
    with torch.inference_mode():
        test_pred = model(test_data)
        loss_test = loss_fn(test_pred, test_target)

    if epoch%(epochs//min(10,epochs))==0: print(f"{epoch} - training loss: {round(float(loss),4)} | test loss: {round(float(loss_test),4)}")- training loss: {loss} | test loss: {loss_test}")

Output
0 - training loss: 0.0837 | test loss: 0.0806
100 - training loss: 0.0433 | test loss: 0.0431
200 - training loss: 0.0426 | test loss: 0.0425
300 - training loss: 0.042 | test loss: 0.0419
400 - training loss: 0.0414 | test loss: 0.0414
500 - training loss: 0.0408 | test loss: 0.0408
600 - training loss: 0.0403 | test loss: 0.0403
700 - training loss: 0.0398 | test loss: 0.0398
800 - training loss: 0.0393 | test loss: 0.0394
900 - training loss: 0.0388 | test loss: 0.0389

Output:
First, as it was mentioned in the comments, you probably meant:
train_data, train_target = data[:int(a*len(data))] , target[:int(a*len(data))]
test_data, test_target = data[int(a*len(data)):] , target[int(a*len(data)):]

Next, your target size is not consistent with the output size (this should give a warning). Using
loss = loss_fn(train_pred, train_target.unsqueeze(1))

and

loss_test = loss_fn(test_pred, test_target.unsqueeze(1))

should give you some traction.


Instruction: Runtime error generated by pytorch functions

Input:
Let us consider following code:
from multiprocessing import freeze_support
import torch
import torch.nn as nn
import torchvision
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
from torchvision import datasets,models, transforms
import time
import os
import copy
import matplotlib.pyplot as plt
#import torch.backends.cudnn as cudnn
#cudnn.benchmark = True
plt.ion()   # interactive mode
path ='C:/Users/User/PycharmProjects/AI_Project/hymenoptera_data'
data_transforms ={
    'train':transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val':transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}
image_datasets ={x:datasets.ImageFolder(os.path.join(path,x),data_transforms[x])
                    for x in ['train','val']}
dataloaders ={x: torch.utils.data.DataLoader(image_datasets[x],batch_size=4,shuffle=True,num_workers=4)
                        for x in ['train','val']}
class_names = image_datasets['train'].classes
device =torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def imshow(inp,title=None):
    inp =inp.numpy().transpose((1,2,0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    inp = np.clip(inp, 0, 1)
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a b
inputs,classes =next((iter(dataloaders['train'])))
out =torchvision.utils.make_grid(inputs)
imshow(out,title=[class_names[x]for x in classes])

When I run following code, I got this error
raise RuntimeError('''
RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.

I did not get point from where this error is coming, maybe DataLoader  causes this error, but how to fix?

Output:
You need to run code with multiple workers (like your DataLoader) in the main function. This is so child processes know whether to join or not.
import torch
import torch.nn as nn
import torchvision
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
from torchvision import datasets,models, transforms
import time
import os
import copy
import matplotlib.pyplot as plt
#import torch.backends.cudnn as cudnn
#cudnn.benchmark = True
plt.ion()   # interactive mode
path ='C:/Users/User/PycharmProjects/AI_Project/hymenoptera_data'

if __name__ == "__main__":
    data_transforms ={
        'train':transforms.Compose([
            transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
        'val':transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
    }
    image_datasets ={x:datasets.ImageFolder(os.path.join(path,x),data_transforms[x])
                     for x in ['train','val']}
    dataloaders ={x: torch.utils.data.DataLoader(image_datasets[x],batch_size=4,shuffle=True,num_workers=4)
                         for x in ['train','val']}
    class_names = image_datasets['train'].classes
    device =torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    def imshow(inp,title=None):
        inp =inp.numpy().transpose((1,2,0))
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])
        inp = std * inp + mean
        inp = np.clip(inp, 0, 1)
        plt.imshow(inp)
        if title is not None:
            plt.title(title)
        plt.pause(0.001)  # pause a b
    inputs,classes =next((iter(dataloaders['train'])))
    out =torchvision.utils.make_grid(inputs)
    imshow(out,title=[class_names[x]for x in classes])


Instruction: Best way to store seed vectors from pytorch?

Input:
I like to make a pytorch calculation reproducible without storing a large random vector for each step. I tried to first generate a random seed and then re-seed the random number generator like this:
    seed = torch.rand(1, dtype=torch.float64)
    torch.manual_seed(seed) # re-seed, so we get the same vector as we would get when using a stored seed
    torch.save(seed, "seedfile") # store the seed
    myvector = torch.randn(myvector.shape)

This way I would only need to store a float to reproduce the result. But when I use this in a loop, I get always the same result inside the loop.
Explanation what I try to achieve: Let's say I generate a batch of images in a loop. Each image depends on an initialization vector. Now I can reproduce the image by storing the initialization vector and loading it when I want to re-do the calculation (e.g. with other hyper-parameters). But when the vector is random anyway, it is sufficient to store the random seed.
To do so, I currently generate a random seed (a float64 in that code) and then manually seed with it. The manual_seed is not useful in the first run, but should not be a problem either. When I want to reproduce the image, I do not generate the manual seed with torch.rand, but load the seed from a file. This way I need less than 1 kb (with torch.save which has some overhead, the actual data I need to store would be just 8 byte) instead of, e.g., 64 kb for storing the vector that is generated by
loaded_seed = torch.load("seedfile")
torch.manual_seed(loaded_seed)
myvector = torch.randn(myvector.shape)

Output: It looks like the problem was (re-)seeding with torch.rand(1) instead of torch.random.seed().


Instruction: Why does Softmax(dim=0) produce poor results?

Input:
I'm getting weird results from a PyTorch Softmax layer, trying to figure out what's going on, so I boiled it down to a minimal test case, a neural network that just learns to decode binary numbers into one-hot form.
Just Softmax() gets a warning:

UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.

Okay, so what to supply for X? I had been guessing 0 would be a sensible argument. Just to make sure, I tried Softmax(dim=1):

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

Okay, so that seems clear about allowed values. -1 apparently means the last dimension, so in this case, where the output is just a one-dimensional vector, that should mean the same thing as 0. Trying it with Softmax(dim=-1) works fine; in a few thousand epochs, the network reliably learns to decode the numbers with 100% accuracy.
Just to make sure it gives the same results, I tried it again with Softmax(dim=0) (as shown below)...
And it does not give the same result at all. The accuracy oscillates, but levels off somewhere around 20-30%.
What's going on? Why is 0 not the same as -1 in this context, and what exactly is 0 doing?
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader

bits = 5

class Dataset1(Dataset):
    def __init__(self):
        s = []
        for i in range(1 << bits):
            x = []
            for c in format(i, "b").zfill(bits):
                x.append(float(c == "1"))

            y = []
            for j in range(1 << bits):
                y.append(float(i == j))

            x = torch.as_tensor(x)
            y = torch.as_tensor(y)
            s.append((x, y))
        self.s = s

    def __len__(self):
        return len(self.s)

    def __getitem__(self, i):
        return self.s[i]

trainDs = Dataset1()

batchSize = 16
trainDl = DataLoader(trainDs, batch_size=batchSize)

for x, y in trainDl:
    print(x.shape)
    print(y.shape)
    break

hiddenSize = 100

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.layers = nn.Sequential(
            nn.Linear(bits, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.Tanh(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, 1 << bits),
            nn.Softmax(dim=0),
        )

    def forward(self, x):
        return self.layers(x)

device = torch.device("cpu")
model = Net().to(device)

def accuracy(model, ds):
    n = 0
    for x, y in ds:
        with torch.no_grad():
            z = model(x)
        if torch.argmax(y) == torch.argmax(z):
            n += 1
    return n / len(ds)

criterion = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
epochs = 10000
interval = epochs // 10
for epoch in range(epochs + 1):
    for bi, (x, y) in enumerate(trainDl):
        x = x.to(device)
        y = y.to(device)

        loss = criterion(model(x), y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if epoch % interval == 0 and not bi:
            print(f"{epoch}\t{loss}\t{accuracy(model, trainDs)}")

Output:
In the accuracy function, you forgot to create a new dimension for the batch (batchsize=1), which explains why it gives that error when you use dim=1. Regarding the dimension of the softmax, you can check this post.
Below is the modified code.
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader

bits = 5

class Dataset1(Dataset):
    def __init__(self):
        s = []
        for i in range(1 << bits):
            x = []
            for c in format(i, "b").zfill(bits):
                x.append(float(c == "1"))

            y = []
            for j in range(1 << bits):
                y.append(float(i == j))

            x = torch.as_tensor(x)
            y = torch.as_tensor(y)
            s.append((x, y))
        self.s = s

    def __len__(self):
        return len(self.s)

    def __getitem__(self, i):
        return self.s[i]

trainDs = Dataset1()

batchSize = 16
trainDl = DataLoader(trainDs, batch_size=batchSize, drop_last=True)

for x, y in trainDl:
    print(x.shape)
    print(y.shape)
    break

hiddenSize = 100

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.layers = nn.ModuleList(
            [nn.Linear(bits, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.Tanh(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, 1 << bits),
            nn.Softmax(dim=1)]
        )

    def forward(self, x):
        for i,layer in enumerate(self.layers):
            x = layer(x)
            if i == 6:
                pass
                #print('softmax input shape',x.shape)
                #print('softmax output shape',torch.nn.functional.softmax(x,dim=1).shape)
                #print('linear',x.shape)
        #print('output',x.shape)
        return x

device = torch.device("cpu")
model = Net().to(device)

def accuracy(model, ds):
    n = 0
    for x, y in ds:
        x = x.unsqueeze(0) # create a batch of size 1
        y = y.unsqueeze(0) # create a batch of size 1
        with torch.no_grad():
            z = model(x)
            print(z.shape)
            break
        if torch.argmax(y) == torch.argmax(z):
            n += 1
    return n / len(ds)

criterion = nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
epochs = 10000
interval = epochs // 10
for epoch in range(epochs + 1):
    for bi, (x, y) in enumerate(trainDl):
        x = x.to(device)
        y = y.to(device)

        loss = criterion(model(x), y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if epoch % interval == 0 and not bi:
            print(f"{epoch}\t{loss}\t{accuracy(model, trainDs)}")


Instruction: Setting torch.nn.linear() diagonal elements zero

Input:
I am trying to build a model with a layer of torch.nn.linear with same input size and output size, so the layer would be square matrix. In this model, I want the diagonal elements of this matrix fixed to zero. Which means, during training, I don't want the diagonal elements to be changed from zero. I could only think of adding some kind of step that change diag elements to zero for each training epoch, but I am not sure if this is valid or efficient way. Is there a definite way of making this kind of layer which can ensure that the diagonal elements don't change?
Sorry if my question is weird.

Output:
You can always implement your own layers. Note that all custom layers should be implemented as classes derived from nn.Module. For example:
class LinearWithZeroDiagonal(nn.Module):
  def __init__(self, num_features, bias):
    super(LinearWithZeroDiagonal, self).__init__()
    self.base_linear_layer = nn.Linear(num_features, num_features, bias)

  def forward(self, x):
    # first, make sure the diagonal is zero
    with torch.no_grad():
      self.base_linear_layer.weight.fill_diagonal_(0.)
    return self.base_linear_layer(x)


Instruction: Trying to run imagen-pytorch

Input:
I'm trying to run this "imagen-pytorch" repo https://github.com/lucidrains/imagen-pytorch/blob/main/README.md
I've cloned it and tried running the setup.py file on pycharm (CE) however I get the following error:
$ python setup.py
usage: setup.py [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...]
   or: setup.py --help [cmd1 cmd2 ...]
   or: setup.py --help-commands
   or: setup.py cmd --help

There are no further instructions on the README.md file. Please can you explain where I am going wrong?

Output:
You have to run with some command.
The most popular are
python setup.py build
python setup.py install

You need them to install module.

You can see other commands using
python setup.py --help-commands

Maybe they forgot to show this information because other modules on GitHub already show it ;)


Instruction: Does equal probabilities not summing to one in torch.utils.data.WeightedRandomSampler still make it uniform?

Input:
In pytorch, there is a sampler class called WeightedRandomSampler (https://pytorch.org/docs/stable/data.html#torch.utils.data.WeightedRandomSampler). It ('weights' parameter) expects probabilities for N samples. For uniform distribution, I believe it expects array with 1/N value.
But if I put say 0.5 for each sample, where N*0.5 is not equal to 1, does it still make the sampling uniform, given equal probabilities are there for each sample?

Output:
Yes, the sampling will still be uniform. Only the relative magnitude of the weights with respect to the other weights is important, not the absolute magnitude, as pytorch normalizes the weights.
If we look under the hood of WeightedRandomSampler, it makes a call to torch.multinomial which itself makes a call to torch.distributions.Categorical, which we can see here (line 57) normalizes the weights such that they sum to one.


Instruction: Get image name of pytorch dataset

Input:
I am using a custom dataset for image segmentation. While visualizing some of the images and masks i found an error. The problem for me know is, how to find the name of the image. The code i use for the pytorch datasetset creation is:
class SegmentationDataset(Dataset):

  def __init__(self, df, augmentations):
    self.df = df
    self.augmentations = augmentations

  def __len__(self):
    return len(self.df)

  def __getitem__(self, idx):
    row = self.df.iloc[idx]

    image_path = DATA_DIR + row.images
    mask_path = DATA_DIR + row.masks

    image = skimage.io.imread(image_path)

    mask = skimage.io.imread(mask_path)
    mask = np.expand_dims(mask, axis = -1)

    if self.augmentations:
      data = self.augmentations(image = image, mask = mask)
      image = data['image']
      mask = data['mask']

    image = np.transpose(image, (2, 0, 1)).astype(np.float32)
    mask = np.transpose(mask, (2, 0, 1)).astype(np.float32)

    image = torch.Tensor(image) / 255.0
    mask = torch.round(torch.Tensor(mask) / 255.0)

    return image, mask

trainset = SegmentationDataset(train_df, get_train_augs())
validset = SegmentationDataset(valid_df, get_valid_augs())

When i then print one specific image, i see that the mask is not available/wrong:
idx = 9
print('Drawn sample ID:', idx)

image, mask = validset[idx]
show_image(image, mask)

How do i now get the image name of this idx = 9?

Output:
I'd imagine you could print out one of the following, under this line  image = skimage.io.imread(image_path), it should help lead you to your answer:
print(row)
print(row.images)
print(images)
print(image_path)

To get the file name after you have parsed the fully quaified path above:
my_str = '/my/data/path/images/wallpaper.jpg'

result = my_str.rsplit('/', 1)[1]
print(result)  # 'wallpaper.jpg'

with_slash = '/' + my_str.rsplit('/', 1)[1]
print(with_slash)  # '/wallpaper.jpg'

['/my/data/path/images/', 'wallpaper.jpg']
print(my_str.rsplit('/', 1)[1])


Instruction: Is it possible that the inference time is large while number of parameters and flops are low in pytorch?

Input:
I calculated flops of network using Pytorch.
I used the function 'profile' in 'thop' library.
In my experiment. My network showed that
Flops : 619.038M
Parameters : 4.191M
Inference time : 25.911
Unlike my experiment, I would check the flops and parameters with ResNet50 which showed that
Flops : 1.315G
Parameters: 26.596M
Inference time : 8.553545
Is is possible that the inference time is large while flops are low?
Or are there flops that the 'profile' function can't measure some functions?
However, Similar results came out using the 'FlopCountAnalysis in fvcore.nn' and 'get_model_complexity_info in ptflops'
Here is the code that I measured the inference time using Pytorch.
model.eval()
model.cuda()

dummy_input = torch.randn(1,3,32,32).cuda()

#flops = FlopCountAnalysis(model, dummy_input)
#print(flop_count_table(flops))
#print(flops.total())

macs, params = profile(model, inputs=(dummy_input,))
macs, params = clever_format([macs, params], "%.3f")
print('Flops:',macs)
print('Parameters:',params)

starter, ender = torch.cuda.Event(enable_timing=True),
torch.cuda.Event(enable_timing=True)

repetitions = 300
timings=np.zeros((repetitions,1))

for _ in range(10):
    _ = model(dummy_input)

# MEASURE PERFORMANCE
with torch.no_grad():
    for rep in range(repetitions):
        starter.record()
        _ = model(dummy_input)
        ender.record()
        # WAIT FOR GPU SYNC
        torch.cuda.synchronize()
        curr_time = starter.elapsed_time(ender)
        timings[rep] = curr_time

print('time(s) :',np.average(timings))

Output:
It is absolutely normal situation. The thing is FLOPS (or MACs) are theoretical measures that may be useful when you want to disregard some hardware/software optimizations that leads to the fact that different operations will work faster/slower on different hardware.
For example in the case of neural networks the different architectures will have different CPU/GPU utilization. Let's consider two simple architectures with almost same number of parameters / FLOPs:

Deep network:

layers = [nn.Conv2d(3, 16, 3)]
for _ in range(12):
    layers.extend([nn.Conv2d(16, 16, 3, padding=1)])
deep_model = nn.Sequential(*layers)

Wide network
wide_model = nn.Sequential(nn.Conv2d(3, 1024, 3))

Modern GPUs allows you to parallelize a large number of simple operations.
But when you have deep network you need to know outputs of layer[i] to compute the outputs of layer[i+1]. So it became blocking factor that reduces utilization of your hardware.
Complete example:
import numpy as np
import torch
from thop import clever_format, profile
from torch import nn

def measure(model, name):
    model.eval()
    model.cuda()

    dummy_input = torch.randn(1, 3, 64, 64).cuda()

    macs, params = profile(model, inputs=(dummy_input,), verbose=0)
    macs, params = clever_format([macs, params], "%.3f")
    print("<" * 50, name)
    print("Flops:", macs)
    print("Parameters:", params)

    starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(
        enable_timing=True
    )

    repetitions = 300
    timings = np.zeros((repetitions, 1))

    for _ in range(10):
        _ = model(dummy_input)

    # MEASURE PERFORMANCE
    with torch.no_grad():
        for rep in range(repetitions):
            starter.record()
            _ = model(dummy_input)
            ender.record()
            # WAIT FOR GPU SYNC
            torch.cuda.synchronize()
            curr_time = starter.elapsed_time(ender)
            timings[rep] = curr_time

    print("time(ms) :", np.average(timings))

layers = [nn.Conv2d(3, 16, 3)]
for _ in range(12):
    layers.extend([nn.Conv2d(16, 16, 3, padding=1)])
deep_model = nn.Sequential(*layers)
measure(deep_model, "My deep model")

wide_model = nn.Sequential(nn.Conv2d(3, 1024, 3))
measure(wide_model, "My wide model")

Results:
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< My deep model
Flops: 107.940M
Parameters: 28.288K
time(ms) : 0.6160109861691793
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< My wide model
Flops: 106.279M
Parameters: 28.672K
time(ms) : 0.1514971748739481

As you can see the models have similar number of parameters/flops but computing time is 4x large for the deep network.
It is just a one of possible reason why the inference time is large when number of parameters and flops are low. You may need to take into account other underlying hardware/software optimizations.


Instruction: How to redefine model for Pytorch max pool error

Input:
Hi I am trying to use tensor board for the model given at - https://github.com/siddharthvaria/WordPair-CNN/blob/master/src/pytorch_impl/cnn_pdtb_arg_multiclass_jl.py
However I get an error at Line 135
: x_max_pools = [F.max_pool1d(xi, xi.size(2)).squeeze(2) for xi in x_convs]
As
max_pool1d(): argument ‘kernel_size’ must be tuple of ints. Is there any solution to resolve this?
There is a discussion about this in pytorch forum - https://discuss.pytorch.org/t/typeerror-avg-pool2d-argument-kernel-size-must-be-tuple-of-ints-not-proxy/108482

Output:
The line that raises error should be line 135:
# At this point x_convs is [(batch_size, nfmaps_arg, seq_len_new), ...]*len(fsz_arg)
x_max_pools = [F.max_pool1d(xi, xi.size(2)).squeeze(2) for xi in x_convs]  # [(batch_size, nfmaps_arg), ...]*len(fsz_arg)

Actually max_pool1d reduces the size of the dimension n°2 that represents the sequence (after batch and channel dimension), see the documentation. Plus, according to the discuss you point out, the kernel_size argument now requires a concrete value at execution and cannot be inferred dynamically from inputs.
Fortunately your case is particularly simple because the kernel size is the entire sequence length. In other words, your pooling is actually a global max pooling that reduces the size of the sequence to only one scalar. This scalar is simply the maximum of all the values (check the shapes in comments!). So it is actually easy to implement a version without dynamic shapes. Just change line 135 by:
x_max_pools = [torch.max(xi, dim=2, keepdim=True)[0] for xi in x_convs]  # [(batch_size, nfmaps_arg), ...]*len(fsz_arg)

Note:

there is no direct implementation of global_max_pooling layer but as you can see it is just a torch.max
Actually torch.max return a tuple (max, max_indicies) so you hae to take only the first element with [0] (see the doc)
gradients can propagate through torch.max like usual max_pooling layers (one source see here for instance)
torch.max(x, dim=2).squeeze(2) is changed to torch.max(x, dim=2, keepdim=True) that is exactly the same thing but cleaner.


Instruction: CrossEntropyLoss showing poor accuracy on 2d output

Input:
I'm trying some experiments on a simple neural network that just tries to learn the squares of some random numbers, represented as arrays of decimal digits, code copied below, with changes indicated by comments.
The version using nn.Softmax(dim=2) and criterion = nn.BCELoss() works fine.
But for situations like this, where the output is N-way classification (in this case, an array of outputs, each of which indicates one of ten decimal digits), CrossEntropyLoss is considered ideal, so I made that change. nn.CrossEntropyLoss does softmax for you, so I also commented out the nn.Softmax line.
And instead of performing slightly better, the result performs much worse; it now maxes out around something like 76% accuracy on the training set, where previously it had reached 100%.
What am I doing wrong? The same substitution worked fine on an even simpler test case https://github.com/russellw/ml/blob/main/compound_output/single.py with the main difference being that case produces only a single N-way output whereas this produces an array of them. Am I misunderstanding how CrossEntropyLoss handles shapes, or some such?
import random
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader

def oneHot(n, i, s):
    for j in range(n):
        s.append(float(i == j))

size = 12

class Dataset1(Dataset):
    def __init__(self):
        s = []
        for _ in range(1000):
            a = random.randrange(10 ** size)

            x = []
            for c in str(a).zfill(size):
                oneHot(10, int(c), x)

            y = []
            for c in str(a ** 2).zfill(size * 2):
                y1 = []
                oneHot(10, int(c), y1)
                y.append(y1)

            x = torch.as_tensor(x)
            y = torch.as_tensor(y)
            s.append((x, y))
        self.s = s

    def __len__(self):
        return len(self.s)

    def __getitem__(self, i):
        return self.s[i]

trainDs = Dataset1()
testDs = Dataset1()

batchSize = 20
trainDl = DataLoader(trainDs, batch_size=batchSize)
testDl = DataLoader(testDs, batch_size=batchSize)
for x, y in trainDl:
    print(x.shape)
    print(y.shape)
    break

class View(nn.Module):
    def __init__(self, *shape):
        super(View, self).__init__()
        self.shape = shape

    def forward(self, x):
        batchSize = x.data.size(0)
        shape = (batchSize,) + self.shape
        return x.view(*shape)

hiddenSize = 100

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.layers = nn.Sequential(
            nn.Linear(size * 10, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.Tanh(),
            nn.Linear(hiddenSize, hiddenSize),
            nn.ReLU(),
            nn.Linear(hiddenSize, size * 2 * 10),
            View(size * 2, 10),
            #nn.Softmax(dim=2),
        )

    def forward(self, x):
        return self.layers(x)

device = torch.device("cpu")
model = Net().to(device)
print(sum(p.numel() for p in model.parameters()))

def accuracy(model, ds):
    n = 0
    for x, y in ds:
        # make input sample shape match a mini batch
        # for the sake of things like softmax that cause the model
        # to expect a specific shape
        x = x.unsqueeze(0)

        # this is just for reporting, not part of training
        # so we don't need to track gradients here
        with torch.no_grad():
            z = model(x)

            # conversely, the model will return a batch-shaped output
            # so unwrap it for comparison with the unwrapped expected output
            z = z[0]

        # at this point, if the output were a scalar mapped to one-hot
        # we could use a simple argmax comparison
        # but it is an array thereof
        # which makes comparison a little more complex
        assert y.shape[0] == size * 2
        assert z.shape[0] == size * 2
        for i in range(0, size * 2):
            if torch.argmax(y[i]) == torch.argmax(z[i]):
                n += 1
    return n / (len(ds) * size * 2)

#criterion = nn.BCELoss()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
epochs = 10000
interval = epochs // 10
for epoch in range(epochs + 1):
    for bi, (x, y) in enumerate(trainDl):
        x = x.to(device)
        y = y.to(device)

        loss = criterion(model(x), y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if epoch % interval == 0 and not bi:
            print(
                f"{epoch}\t{loss}\t{accuracy(model, trainDs)}\t{accuracy(model, testDs)}"
            )

Output:
Even if it doesn't raise error, torch.BCELoss is not actually what you want to minimize because it wrongly interprets your tensors as a multitude of binary classifications. Therefore it is a good idea to switch to torch.nn.CrossEntropyLoss.
As you can see in its documentation the function takes class number at targets (not one-hot-encoded) and only supports tensor with at most one dimension for the batch. So you can try:
x = x.to(device)
y = y.to(device)
# Flat together the figures prediction in the batch
pred = model(x).reshape(-1, 10)  # shape (batch_size*2*size , 10)
# Reverse one-hot encoding for targets + flat
y = torch.argmax(y, dim=2).reshape(-1) # shape (batch_size*2*size, )
loss = criterion(pred, y)

I got 100% training accuracy at epoch 1100 with your config (same architecture, CPU, batch size 20) as you can see:

Note that the model actually overfits the training data in this scenario but it is an other problem...


Instruction: TypeError: dropout(): argument 'input' (position 1) must be Tensor, not tuple

Input:
I am studying NLP and trying to make a model for classifying sentences.  I am creating my class with a model but I get an error saying that the input should be of type Tensor, not tuple. I use 4.21.2 transformers version.
class BertClassificationModel(nn.Module):
    def __init__(self, bert_model_name, num_labels, dropout=0.1):
        super(BertClassificationModel, self).__init__()
        self.bert = BertForSequenceClassification.from_pretrained(bert_model_name, return_dict=False)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(768, num_labels)
        self.num_labels = num_labels
    def forward(self, input_ids, attention_mask=None, token_type_ids=None):
        pooled_output = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
        return logits

TypeError: dropout(): argument 'input' (position 1) must be Tensor, not tuple

Output:
The issue you face, is that the output of self.bert is not a tensor but a tuple:
from transformers import BertForSequenceClassification, BertTokenizer

bert_model_name = "bert-base-cased"
t = BertTokenizer.from_pretrained(bert_model_name)
m = BertForSequenceClassification.from_pretrained(bert_model_name, return_dict=False)

o=m(**t("test test", return_tensors="pt"))

print(type(o))

Output:
tuple

I personally do not recommend using return_dict=False as the code becomes more difficult to read. But changing this parameter doesn't help in your case, as you want to use the pooler output which is removed by the classification head of BertForSequenceClassification (the output of BertForSequenceClassification is listed here).
You already wrote in your own answer, that you don't intend to use the classification head of BertForSequenceClassification and you can therefore load BertModel directly (instead of initializing BertForSequenceClassification and only using BERT as you did with: BertForSequenceClassification.from_pretrained(bert_model_name, return_dict=True).bert):
from torch import nn
from transformers import BertModel, BertTokenizer

class BertClassificationModel(nn.Module):
    def __init__(self, bert_model_name, num_labels, dropout=0.1):
        super(BertClassificationModel, self).__init__()
        self.bert = BertModel.from_pretrained(bert_model_name)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(768, num_labels)
        self.num_labels = num_labels
    def forward(self, input_ids, attention_mask=None, token_type_ids=None):
        pooled_output = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids).pooler_output
        pooled_output = self.dropout(pooled_output)
        logits = self.classifier(pooled_output)
        return logits

m = BertClassificationModel("bert-base-cased",4, 0.1)
o = m(**t("test test", return_tensors="pt"))
print(o.shape)

Output:
torch.Size([1, 4])


Instruction: What is the reason for low GPU util when training machine learning model?

Input:
Suppose I have 8 gpus on a server. (From 0 to 7)
When I train a simple (and small) model on a gpu #0, it takes about 20 minutes per epoch.
However, when I load more than 5 or 6 models on some gpus,
for example, 2 experiments per gpu from gpu #0 to #2, (6 in total)
the training time per epoch explodes. ( about 1 hour ),
When I train 2 models per gpu for all gpus ( 16 experiments in total ), it takes about 3 hours to complete an epoch.
When I see the CPU utilization, it is fine.
But GPU utilization drops.
What is the reason for the drop, and how can I solve the problem?

Output:
There are basically two ways of using multi-GPUs for deep learning:

Use torch.nn.DataParallel(module) (DP)

This function is quite discouraged by the official documentation because it replicates the entire module in all GPUs at each forward pass. At the end of forward pass, the models are destroyed. Therefore, when you have big models it could be an important bottleneck in your training time and even slow it by compared to single GPU. It could be the case for instance when you freeze a large part of big module for fine tuning.
That's why you may consider using:

torch.nn.parallel.DistributedDataParallel(module, device_ids=) (DDP) documentation

This function often requires refactoring your code a little bit more but it improves the efficiency because it copy the models on GPUs only once, at the beginning of the training. The models are persistent over time and the gradients are synchronized after each backward pass via hooks. To go further, you can distributed data and optimizer as well to avoid data transfer. You can do it simply (as well as parallelized modules) using torch-ignite/distributed.
I don't know what kind of method you tried but I encourage you to use DDP instead of DP if you are using it.


Instruction: 400% higher error with PyTorch compared with identical Keras model (with Adam optimizer)

Input:
TLDR:
A simple (single hidden-layer) feed-forward Pytorch model trained to predict the function y = sin(X1) + sin(X2) + ... sin(X10) substantially underperforms an identical model built/trained with Keras. Why is this so and what can be done to mitigate the difference in performance?

In training a regression model, I noticed that PyTorch drastically underperforms an identical model built with Keras.
This phenomenon has been observed and reported previously:

The same model produces worse results on pytorch than on tensorflow

CNN model in pytorch giving 30% less accuracy to Tensoflowflow model:

PyTorch Adam vs Tensorflow Adam

Suboptimal convergence when compared with TensorFlow model

RNN and Adam: slower convergence than Keras

PyTorch comparable but worse than keras on a simple feed forward network

Why is the PyTorch model doing worse than the same model in Keras even with the same weight initialization?

Why Keras behave better than Pytorch under the same network configuration?

The following explanations and suggestions have been made previously as well:

Using the same decimal precision (32 vs 64): 1, 2,

Using a CPU instead of a GPU: 1,2

Change retain_graph=True to create_graph=True in computing the 2nd derivative with autograd.grad: 1

Check if keras is using a regularizer, constraint, bias, or loss function in a different way from pytorch: 1,2

Ensure you are computing the validation loss in the same way: 1

Use the same initialization routine: 1,2

Training the pytorch model for longer epochs: 1

Trying several random seeds: 1

Ensure that model.eval() is called in validation step when training pytorch model: 1

The main issue is with the Adam optimizer, not the initialization: 1

To understand this issue, I trained a simple two-layer neural network (much simpler than my original model) in Keras and PyTorch, using the same hyperparameters and initialization routines, and following all the recommendations listed above. However, the PyTorch model results in a mean squared error (MSE) that is  400% higher than the MSE of the Keras model.
Here is my code:
0. Imports
import numpy as np
from scipy.stats import pearsonr

from sklearn.preprocessing import MinMaxScaler
from sklearn import metrics

from torch.utils.data import Dataset, DataLoader

import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.regularizers import L2
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam

1. Generate a reproducible dataset

def get_data():

    np.random.seed(0)
    Xtrain = np.random.normal(0, 1, size=(7000,10))
    Xval = np.random.normal(0, 1, size=(700,10))
    ytrain = np.sum(np.sin(Xtrain), axis=-1)
    yval = np.sum(np.sin(Xval), axis=-1)
    scaler = MinMaxScaler()
    ytrain = scaler.fit_transform(ytrain.reshape(-1,1)).reshape(-1)
    yval = scaler.transform(yval.reshape(-1,1)).reshape(-1)

    return Xtrain, Xval, ytrain, yval

class XYData(Dataset):

    def __init__(self, X, y):

        super(XYData, self).__init__()
        self.X = torch.tensor(X, dtype=torch.float32)
        self.y = torch.tensor(y, dtype=torch.float32)
        self.len = len(y)

    def __getitem__(self, index):

        return (self.X[index], self.y[index])

    def __len__(self):

        return self.len

# Data, dataset, and dataloader
Xtrain, Xval, ytrain, yval = get_data()
traindata = XYData(Xtrain, ytrain)
valdata = XYData(Xval, yval)
trainloader = DataLoader(dataset=traindata, shuffle=True, batch_size=32, drop_last=False)
valloader = DataLoader(dataset=valdata, shuffle=True, batch_size=32, drop_last=False)

2. Build Keras and PyTorch models with identical hyperparameters and initialization methods
class TorchLinearModel(nn.Module):

    def __init__(self, input_dim=10, random_seed=0):

        super(TorchLinearModel, self).__init__()
        _ = torch.manual_seed(random_seed)
        self.hidden_layer = nn.Linear(input_dim,100)
        self.initialize_layer(self.hidden_layer)
        self.output_layer = nn.Linear(100, 1)
        self.initialize_layer(self.output_layer)

    def initialize_layer(self, layer):

        _ = torch.nn.init.xavier_normal_(layer.weight)
        #_ = torch.nn.init.xavier_uniform_(layer.weight)
        _ = torch.nn.init.constant(layer.bias,0)

    def forward(self, x):
        x = self.hidden_layer(x)
        x = self.output_layer(x)
        return x

def mean_squared_error(ytrue, ypred):

    return torch.mean(((ytrue - ypred) ** 2))

def build_torch_model():

    torch_model = TorchLinearModel()
    optimizer = optim.Adam(torch_model.parameters(),
                           betas=(0.9,0.9999),
                           eps=1e-7,
                           lr=1e-3,
                           weight_decay=0)
    return torch_model, optimizer

def build_keras_model():

    x = layers.Input(shape=10)
    z = layers.Dense(units=100, activation=None, use_bias=True, kernel_regularizer=None,
                     bias_regularizer=None)(x)
    y = layers.Dense(units=1, activation=None, use_bias=True, kernel_regularizer=None,
                     bias_regularizer=None)(z)
    keras_model = Model(x, y, name='linear')
    optimizer = Adam(learning_rate=1e-3, beta_1=0.9, beta_2=0.9999, epsilon=1e-7,
                     amsgrad=False)

    keras_model.compile(optimizer=optimizer, loss='mean_squared_error')

    return keras_model

# Instantiate models
torch_model, optimizer = build_torch_model()
keras_model = build_keras_model()

3. Train PyTorch model for 100 epochs:

torch_trainlosses, torch_vallosses = [], []

for epoch in range(100):

    # Training
    losses = []
    _ = torch_model.train()

    for i, (x,y) in enumerate(trainloader):
        optimizer.zero_grad()
        ypred = torch_model(x)
        loss = mean_squared_error(y, ypred)
        _ = loss.backward()
        _ = optimizer.step()
        losses.append(loss.item())
    torch_trainlosses.append(np.mean(losses))

    # Validation
    losses = []
    _ = torch_model.eval()

    with torch.no_grad():
        for i, (x, y) in enumerate(valloader):
            ypred = torch_model(x)
            loss = mean_squared_error(y, ypred)
            losses.append(loss.item())
    torch_vallosses.append(np.mean(losses))

    print(f"epoch={epoch+1}, train_loss={torch_trainlosses[-1]:.4f}, val_loss={torch_vallosses[-1]:.4f}")

4. Train Keras model for 100 epochs:
history = keras_model.fit(Xtrain, ytrain, sample_weight=None, batch_size=32, epochs=100,
                    validation_data=(Xval, yval))

5. Loss in training history
plt.plot(torch_trainlosses, color='blue', label='PyTorch Train')
plt.plot(torch_vallosses, color='blue', linestyle='--', label='PyTorch Val')
plt.plot(history.history['loss'], color='brown', label='Keras Train')
plt.plot(history.history['val_loss'], color='brown', linestyle='--', label='Keras Val')
plt.legend()

Keras records a much lower error in the training. Since this may be due to a difference in how Keras computes the loss, I calculated the prediction error on the validation set with sklearn.metrics.mean_squared_error
6. Validation error after training
ypred_keras = keras_model.predict(Xval).reshape(-1)
ypred_torch = torch_model(torch.tensor(Xval, dtype=torch.float32))
ypred_torch = ypred_torch.detach().numpy().reshape(-1)

mse_keras = metrics.mean_squared_error(yval, ypred_keras)
mse_torch = metrics.mean_squared_error(yval, ypred_torch)
print('Percent error difference:', (mse_torch / mse_keras - 1) * 100)

r_keras = pearsonr(yval, ypred_keras)[0]
r_pytorch = pearsonr(yval, ypred_torch)[0]
print("r_keras:", r_keras)
print("r_pytorch:", r_pytorch)

plt.scatter(ypred_keras, yval); plt.title('Keras'); plt.show(); plt.close()
plt.scatter(ypred_torch, yval); plt.title('Pytorch'); plt.show(); plt.close()

Percent error difference: 479.1312469426776
r_keras: 0.9115184443702814
r_pytorch: 0.21728812737220082

The correlation of predicted values with ground truth is 0.912 for Keras but 0.217 for Pytorch, and the error for Pytorch is 479% higher!
7. Other trials
I also tried:

Lowering the learning rate for Pytorch (lr=1e-4), R increases from 0.217 to 0.576, but it's still much worse than Keras (r=0.912).
Increasing the learning rate for Pytorch (lr=1e-2), R is worse at 0.095
Training numerous times with different random seeds. The performance is roughly the same, regardless.
Trained for longer than 100 epochs. No improvement was observed!
Used torch.nn.init.xavier_uniform_ instead of torch.nn.init.xavier_normal_ in the initialization of the weights. R improves from 0.217 to 0.639, but it's still worse than Keras (0.912).

What can be done to ensure that the PyTorch model converges to a reasonable error comparable with the Keras model?

Output:
The problem here is unintentional broadcasting in the PyTorch training loop.
The result of a nn.Linear operation always has shape [B,D], where B is the batch size and D is the output dimension. Therefore, in your mean_squared_error function ypred has shape [32,1] and ytrue has shape [32]. By the broadcasting rules used by NumPy and PyTorch this means that ytrue - ypred has shape [32,32]. What you almost certainly meant is for ypred to have shape [32]. This can be accomplished in many ways; probably the most readable is to use Tensor.flatten
class TorchLinearModel(nn.Module):
    ...
    def forward(self, x):
        x = self.hidden_layer(x)
        x = self.output_layer(x)
        return x.flatten()

which produces the following train/val curves


Instruction: Forcing PyTorch Neural Net to output a specific datatype

Input:
I am learning how to create a GAN with PyTorch 1.12 and I need the instance returned by my generator to fall into a specific feature space.
The model in my generator class looks like this:
self.model = nn.Sequential(
            nn.Linear(2, 16),
            nn.ReLU(),
            nn.Linear(16, 32),
            nn.ReLU(),
            nn.Linear(32, 2),
        )

I need every feature in the instance returned by my generator to be an unsigned integer. The noise fed into the generator fits this, but the output has decimals and some features are negative. How can I set this?
(If needed I can include more of my code but I figured this is the relevant line and wanted to keep this question short to make things easier for everyone.)

Output:
You can try clipping negative values and casting to torch.int32:
from torch import nn
import torch

class TransformOutput(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.clamp_min(x, min=0).to(dtype=torch.int32)

model = nn.Sequential(
    nn.Linear(2, 16),
    nn.ReLU(),
    nn.Linear(16, 32),
    nn.ReLU(),
    nn.Linear(32, 2),
    TransformOutput(),
)

x = torch.rand(4, 2) - 0.5
out = model(x)
assert out.dtype == torch.int32


Instruction: VGG-16 and ResNet-9 loss values not corresponding to accuracy on test set

Input:
I have two models whose performance I am comparing, a ResNet-9 model and a VGG-16 model. They are being used for image classification. Their accuracies on the same test set are:
ResNet-9 = 99.25% accuracy
VGG-16 = 97.90% accuracy

However from their loss curves during training, shown in the images, I see that the VGG-16 has lower losses as compared to ResNet-9 which has higher losses.
I am using torch.nn.CrossEntropyLoss() for both VGG-16 and ResNet-9. I would have expected the ResNet-9 to have lower losses (because it performs better on the test set) but this is not the case.
Is this observation normal?

Output:
Yes, the loss for a model can be greater even if the accuracy is greater. This is because the loss function penalizes if the confidence is lower.
For example:
if you have a label [0, 0, 0, 1],
and model A predicts [0.1, 0.1, 0.1, 0.7]
and model B predicts [0.1, 0.0, 0.0, 0.9]
Both models will have an accuracy of 100% but the CrossEntropyLoss for model A will be greater than that of model B because model A predicts with a lower confidence score.
So, a higher loss of ResNet-9 compared to VGG-16, even when ResNet-9 has a greater accuracy, means ResNet-9's predictions (on average) have less confidence on the predicted labels.


Instruction: pytorch std must more than or equal to 0.0

Input:
I tried to use torch.normal but got the error that shows std >= 0.0; I need to fix this error.
b=32
n_s = 10
dim = 64
slots_mu = nn.Parameter(torch.randn(1, 1, dim))
slots_log_sigma = nn.Parameter(torch.randn(1, 1, dim))

mu = slots_mu.expand(b, n_s, -1)
sigma = slots_log_sigma.expand(b, n_s, -1)

slots = torch.normal(mu, sigma)

and it raised an error below
---> 10 slots = torch.normal(mu, sigma)

RuntimeError: normal expects all elements of std >= 0.0

Output:
It's because of the definition of standard deviation, std is for distance measurement, please take a look at this answer.
To solve your problem, converting std to absolute value can help:
slots_log_sigma = nn.Parameter(abs(torch.randn(1, 1, dim)))


Instruction: Why the grad is unavailable for the tensor in gpu

Input:
a = torch.nn.Parameter(torch.ones(5, 5))
a = a.cuda()
print(a.requires_grad)
b = a
b = b - 2
print('a ', a)
print('b ', b)
loss = (b - 1).pow(2).sum()
loss.backward()
print(a.grad)
print(b.grad)

After executing codes, the a.grad is None although a.requires_grad is True.
But if the code a = a.cuda() is removed, a.grad is available after the loss backward.

Output:
The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more information.

a = torch.nn.Parameter(torch.ones(5, 5))
a = a.cuda()
print(a.requires_grad)
b = a
b = b - 2
print('a ', a)
print('b ', b)
loss = (b - 1).pow(2).sum()

a.retain_grad() # added this line

loss.backward()
print(a.grad)

That happens because of your line a = a.cuda() that override the original value of a.
You could use
a = torch.nn.Parameter(torch.ones(5, 5))
a.cuda()

Or
a = torch.nn.Parameter(torch.ones(5, 5, device='cuda'))

a = torch.nn.Parameter(torch.ones(5, 5).cuda())

Or explicitly requesting to retain the gradients of a
a.retain_grad() # added this line

Erasing the gradients of intermediate variables can save significant amount of memory. So it is good that you retain gradients only where you need.


Instruction: Pytorch - RuntimeError: Error(s) in loading state_dict for Sequential: Unexpected key(s) in state_dict: "0.weight", "0.bias",

Input:
I have created a Pytorch object from  the class Sequential (see official page).
As they suggest, I am saving it using the command torch.save(model.state_dict(), PATH).
It seems that everything has worked fine, since when I use torch.load(PATH) in another file I get
an ordered Dict like
'0.weight': tensor([[ 0.1202, ...]]) ,
'0.bias':  tensor([ 0.1422, ...]) ,
...

with all the shapes of the tensors being correct. However, when doing
model = Sequential()
model.load_state_dict(torch.load(PATH))

I get the error
RuntimeError: Error(s) in loading state_dict for Sequential:
        Unexpected key(s) in state_dict: "0.weight", "0.bias", "2.weight", "2.bias", "4.weight", "4.bias".

Output:
When trying to load, the model you are trying to load into (model) is an empty Sequential object with no layers. On the other hand, looking at the error message, the state dictionary of the model you are trying to load from indicates that it has at least five layers, with the first, third, and fifth layers containing a weight and bias parameter. This is a mismatch since the corresponding layers and parameters do not exist in model.
To fix this, model should have the same architecture as the saved model you are trying to load. Since you say you saved the original model yourself, use an identical initialization step when creating model.


Instruction: Loading resnet50 prettrianed model in PyTorch

Input:
I want to use resnet50 pretrained model using PyTorch and I am using the following code for loading it:
    import torch
    model = torch.hub.load("pytorch/vision", "resnet50", weights="IMAGENET1K_V2")

Although I upgrade the torchvision but I receive the following error:

Any idea?

Output:
As per the latest definition, we now load models using torchvision library, you can try that using:
from torchvision.models import resnet50, ResNet50_Weights

# Old weights with accuracy 76.130%
model1 = resnet50(weights=ResNet50_Weights.IMAGENET1K_V1)

# New weights with accuracy 80.858%
model2 = resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)


Instruction: Is there any difference between the DNN model by keras and the DNN model by pytorch?

Input:
Here are my codes for DNN by torch and keras.
I use them to train the same data, but finally get the totally different AUC results(keras version reaches 0.74 and torch version reaches 0.67).
So I'm so confused!
And I have tried many times that my results remain differences.
Is there any difference between the two models?
categorical_embed_sizes = [589806, 21225, 2565, 2686, 343, 344, 10, 2, 8, 8, 7, 7, 2, 2, 2, 17, 17, 17]

#keras model
cat_input, embeds = [], []
for i in range(cat_len):
    input_ = Input(shape=(1, ))
    cat_input.append(input_)
    nums = categorical_embed_sizes[i]
    embed = Embedding(nums, 8)(input_)
    embeds.append(embed)
cont_input = Input(shape=(cont_len,), name='cont_input', dtype='float32')
cont_input_r = Reshape((1, cont_len))(cont_input)

embeds.append(cont_input_r)

#Merge_L=concatenate([train_emb,trainnumber_emb,departstationname_emb,arrivestationname_emb,seatname_emb,orderofftime_emb,fromcityname_emb,tocityname_emb,daytype_emb,num_input_r])
Merge_L=concatenate(embeds, name='cat_1')
Merge_L=Dense(256,activation=None,name='dense_0')(Merge_L)
Merge_L=PReLU(name='merge_0')(Merge_L)
Merge_L=BatchNormalization(name='bn_0')(Merge_L)
Merge_L=Dense(128,activation=None,name='dense_1')(Merge_L)
Merge_L=PReLU(name='prelu_1')(Merge_L)
Merge_L=BatchNormalization(name='bn_1')(Merge_L)
Merge_L=Dense(64,activation=None,name='Dense_2')(Merge_L)
Merge_L=PReLU(name='prelu_2')(Merge_L)
Merge_L=BatchNormalization(name='bn_2')(Merge_L)
Merge_L=Dense(32,activation=None,name='Dense_3')(Merge_L)
Merge_L=PReLU(name='prelu_3')(Merge_L)
Merge_L=BatchNormalization(name='bn_3')(Merge_L)
Merge_L=Dense(16,activation=None,name='Dense_4')(Merge_L)
Merge_L=PReLU(name='prelu_4')(Merge_L)

predictions= Dense(1, activation='sigmoid', name='Dense_rs')(Merge_L)

predictions=Reshape((1,), name='pred')(predictions)

cat_input.append(cont_input)
model = Model(inputs=cat_input,
                     outputs=predictions)

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[tf.keras.metrics.BinaryAccuracy(), tf.keras.metrics.AUC()])

# torch model
class DNN(nn.Module):
    def __init__(self, categorical_length, categorical_embed_sizes, categorical_embed_dim, in_size):
        super(DNN, self).__init__()
        self.categorical_length = categorical_length
        self.categorical_embed_sizes = categorical_embed_sizes
        self.categorical_embed_dim = categorical_embed_dim
        self.in_size = in_size
        self.nn = torch.nn.Sequential(
            nn.Linear(self.in_size, 256),
            nn.PReLU(256),
            nn.BatchNorm1d(256),
            nn.Linear(256, 128),
            nn.PReLU(128),
            nn.BatchNorm1d(128),
            nn.Linear(128, 64),
            nn.PReLU(64),
            nn.BatchNorm1d(64),
            nn.Linear(64, 32),
            nn.PReLU(32),
            nn.BatchNorm1d(32),
            nn.Linear(32, 16),
            nn.PReLU(16)
        )
        self.out = torch.nn.Sequential(
            nn.Linear(16, 1),
            nn.Sigmoid()
        )
        self.embedding = nn.Embedding(self.categorical_embed_sizes, self.categorical_embed_dim)

    def forward(self, x):
        x_categorical = x[:, :self.categorical_length].long()
        x_categorical = self.embedding(x_categorical).view(x_categorical.size(0), -1)
        x = torch.cat((x_categorical, x[:, self.categorical_length:]), dim=1)
        x = self.nn(x)
        out = self.out(x)
        return out

Output:
Finally I find the real reason for the error I met. It has no business with the model structure or parameters. Actually, the wrong input for sklearn's roc_auc_score function is the direct cause of this error.
As we know, sklearn.metrics.roc_auc_score need at least y_true and y_score. y_true is the real labels of datasets and y_score is the predicted probabilities of label 1 (for binary tasks).
But when I use torch's outputs to calculate two metrics(Accuracy and AUC), I transform the outputs to 0-1 vectors. So my y_score is no longer probabilities but 0-1 vectors.
Then the error happened...


Instruction: "RuntimeError: a view of a leaf Variable that requires grad is being used in an in-place operation" when there's actually no in-place operations

Input:
I am working on some paper replication, but I am having trouble with it.
According to the log, it says that RuntimeError: a view of a leaf Variable that requires grad is being used in an in-place operation. However, when I check the line where the error is referring to, it was just a simple property setter inside the class:
@pdfvec.setter
def pdfvec(self, value):
   self.param.pdfvec[self.key] = value   # where the error message is referring to

Isn't in-place operations are something like += or *= etc.? I don't see why this error message appeared in this line.
I am really confused about this message, and I will be glad if any one knows any possible reason this can happen.
For additional information, this is the part where the setter function was called:
def _update_params(params, pdfvecs):
    idx = 0
    for param in params:
        totdim = param.stats.numel()
        shape = param.stats.shape
        param.pdfvec = pdfvecs[idx: idx + totdim].reshape(shape)   # where the setter function was called
        idx += totdim

I know this can still lack information for solving the problem, but if you know any possiblity why the error message appeared I would be really glad to hear.

Output:
In-place operation means the assignment you've done is modifiying the underlying storage of your Tensor, of which requires_grad is set to True, according to your error message.
That said, your param.pdfvec[self.key] is not a leaf Tensor, because they will be updated during back-propagation. And you tried to assign a value to it , that will interference with autograd, so this action is prohibited by default. You can do this by directly modifying its underlying storage(f.e., with .data).


Instruction: PyTorch CNN doesn't update weights while training

Input:
I want to predict a 8x8 matrix with the original 8x8 matrix. But the weights DO NOT update in the training process.
I use two simple conv layers to conv input matrix from 1x8x8 to 2x8x8. Then I used another conv layer to convert 2x8x8 to 1x8x8. The inputs and outputs in the data folder are generated randomly. The pytorch codes are shown as follows.
I have already checked some posts about weights not update issues. I think there must be some wrong with "requires_grad = True" of data  or loss.backward().
Any suggestions about the codes would be grateful. Thanks in advance.
                            M
                            Tue Sep  6 15:34:17 CST 2022

The data input folder is in
data/CM10_1/CM_1.txt
data/CM10_1/CM_2.txt
data/CM10_1/CM_3.txt
data/CM10_1/CM_4.txt
The data output folder is in
data/CM10_2/CM_1.txt
data/CM10_2/CM_2.txt
data/CM10_2/CM_3.txt
data/CM10_2/CM_4.txt
CM_i.txt is shown as
207 244 107 173 70 111 180 244
230 246 233 193 11 97 192 86
32 40 202 189 24 195 70 149
232 247 244 100 209 202 173 57
161 244 167 167 177 47 167 191
24 123 9 43 80 124 41 65
71 204 216 180 242 113 30 129
139 36 238 8 8 164 127 178

data/CM_info_tr.csv
CMname,
CM_1.txt,
CM_2.txt,
CM_3.txt,
CM_4.txt,

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# outline###############################################################
#
#  CM10_1/CM_i.txt to predict CM10_2/CM_i.txt
#
#  data pair example
#  CM10_1/CM_1.txt -> CM10_2/CM_1.txt
#
#  CM10_1/CM_1.txt is 8x8 matrix with random int
#  CM10_2/CM_1.txt is 8x8 matrix with random int
#
#  The model uses two conv layers
#  layer 01 : 1x8x8 -> 2x8x8
#  layer 02 : 2x8x8 -> 1x8x8
#
#  The loss is the difference between
#  CM10_2/CM_1.txt(predicted) and CM10_2/CM_1.txt
#

# main ###############################################################

from __future__       import print_function, division

import os
import sys
import torch
import pandas              as pd
import numpy               as np
import torch.nn.functional as F

from skimage          import io, transform

from torch.utils.data import Dataset, DataLoader
from torch            import nn
from torch.autograd   import Variable

torch.cuda.empty_cache()

# Ignore warnings
import warnings
warnings.filterwarnings("ignore")

# test CM parameters
n_Ca   = 8
batch_size = 4

#device = "cuda" if torch.cuda.is_available() else "cpu"
device = "cpu"

# define class dataset CMDataset ###################################################
class CMDataset(Dataset):
    """CM dataset"""
    def __init__(self,csv_CM,CM_beg_dir,CM_end_dir,n_Ca=n_Ca):
        """
        Args:
        csv_CM       (string): Path to the csv file with CM class.
        CM_beg_dir   (string): Directory with all the CM begin data.
        CM_end_dir   (string): Directory with all the CM end data.
        """
        self.CM_info    = pd.read_csv(csv_CM)
        self.CM_beg_dir = CM_beg_dir
        self.CM_end_dir = CM_end_dir

    def __len__(self):
        return len(self.CM_info)# the number of the samples

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        #load and convert CM begin data ---------------------------------------
        CM_beg_path  = os.path.join(self.CM_beg_dir, self.CM_info.iloc[idx, 0])
        CM_beg_data  = np.loadtxt(CM_beg_path)
        CM_beg_data  = CM_beg_data.reshape(1,n_Ca,n_Ca)
        CM_beg_data  = CM_beg_data.astype(np.float32)
        CM_beg_data  = torch.from_numpy(CM_beg_data)
        CM_beg_data  = CM_beg_data.to(device)

        #load and convert CM endin data ---------------------------------------
        CM_end_path  = os.path.join(self.CM_end_dir, self.CM_info.iloc[idx, 0])
        CM_end_data  = np.loadtxt(CM_end_path)
        CM_end_data  = CM_end_data.reshape(1,n_Ca,n_Ca)
        CM_end_data  = CM_end_data.astype(np.float32)
        CM_end_data  = torch.from_numpy(CM_end_data)
        CM_end_data  = CM_end_data.to(device)

        return CM_beg_data, CM_end_data

# define class model CMNet ###################################################
class CMNet(nn.Module):
    def __init__(self):
        super(CMNet, self).__init__()
        self.lay_CM_01 = nn.Conv2d(in_channels=1,out_channels=2,kernel_size=1,stride=1,bias=True)
        self.lay_CM_02 = nn.Conv2d(in_channels=2,out_channels=1,kernel_size=1,stride=1,bias=True)

    def forward(self, CM_data):

        [n_in_batch,n_in_chan,n_in_hei,n_in_wid]=CM_data.shape
        n_Ca = n_in_hei

        out1_1 = self.lay_CM_01(CM_data)
        out1_2 = out1_1
        out1_3 = self.lay_CM_02(out1_2)
        out = out1_3

        return out

# load data for training and validing
CM_dataset_train = CMDataset(csv_CM     = 'data/CM_info_tr.csv',
                             CM_beg_dir = 'data/CM10_1/',
                             CM_end_dir = 'data/CM10_2/',
                             n_Ca       =  n_Ca)

train_dataloader = DataLoader(CM_dataset_train,
                              batch_size=batch_size,
                              shuffle=True)

# training parameter
learning_rate = 2
epochs        = 5

model = CMNet()
model = model.to(device)

# Initialize the loss function
loss_fn = nn.MSELoss(reduction='mean')

optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

# define train loop ###############################################################
def train_loop(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch, (X1,Y) in enumerate(dataloader):
        X1=X1.to(torch.float32)
        Y = Y.to(torch.float32)

        # Compute prediction and loss
        X1=torch.autograd.Variable(X1)
        pred = model(X1)
        pred = torch.autograd.Variable(pred)

        # compute loss
        loss = loss_fn(pred,Y)
        loss = Variable(loss, requires_grad = True)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        loss, current = loss.item(), batch * len(X1)
        print(f" loss:{loss:>15f},  [{current:>5d}/{size:>5d}]")

# Train ###############################################################
for t in range(epochs):
    print(f"Epoch {t+1}\n----------------------------------------------")
#    print(list(model.parameters()))
    train_loop(train_dataloader, model, loss_fn, optimizer)

#print("Train and Valid Done!")

Output: What pytorch version are you using? Variable is depracated for 5 years now. Remove the lines loss = Variable(loss, requires_grad = True) and pred = torch.autograd.Variable(pred), that should do the trick. Try and read the current documentation and don't rely on archaic tutorials.