Instruction: pytorch reduce_op warning message despite not calling it

Input:
I'm constantly receiving a warning message per below; despite not calling the pytorch reduce_op anywhere.
C:\Users\cocoj\.conda\envs\py39\lib\site-packages\torch\distributed\distributed_c10d.py:170: UserWarning: torch.distributed.reduce_op is deprecated, please use torch.distributed.ReduceOp instead
  warnings.warn(

I have found the below link; however not clear what the OP suggested as solution.
https://github.com/ucbrise/flor/issues/57
my pytorch is most uptodate per conda list:
pytorch                   1.10.0          py3.9_cuda11.3_cudnn8_0    pytorch

Output:
I am also not clear on what they meant, but since they were saying that it's safe to ignore you can try using the warnings module to ignore the message like so:
import warnings
warnings.filterwarnings("ignore", message="torch.distributed.reduce_op is deprecated")

Note that it will ignore anything containing the string in the 'message' argument, so use with caution if you don't enter the full error message.
You can also read this question, which has a similar solution.


Instruction: RuntimeError: Found dtype Double but expected Float - Pytorch RL

Input:
I am trying to get an actor critic variant of the pendulum running, but I seem to be running into a particular problem.
RuntimeError: Found dtype Double but expected Float

I saw this had come up multiple times before so I have been through those and have attempted to change the data types of my loss (kept in comments) but it is still not working. Could anyone point out how to resolve this so that I can learn from it?
Full code below
import gym, os
import numpy as np
from itertools import count
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal
from collections import namedtuple

SavedAction = namedtuple('SavedAction', ['log_prob', 'value'])
LOG_SIG_MAX = 2
LOG_SIG_MIN = -20

class ActorCritic(nn.Module):
    """
    Implementing both heads of the actor critic model
    """
    def __init__(self, state_space, action_space):
        super(ActorCritic, self).__init__()

        self.state_space = state_space
        self.action_space = action_space

        # HL 1
        self.linear1 = nn.Linear(self.state_space, 128)
        # HL 2
        self.linear2 = nn.Linear(128, 256)

        # Outputs
        self.critic_head = nn.Linear(256, 1)
        self.action_mean = nn.Linear(256, self.action_space)
        self.action_std = nn.Linear(256, self.action_space)

        # Saving
        self.saved_actions = []
        self.rewards = []

        # Optimizer
        self.optimizer = optim.Adam(self.parameters(), lr = 1e-3)
        self.eps = np.finfo(np.float32).eps.item()

    def forward(self, state):
        """
        Forward pass for both actor and critic
        """

        # State to Layer 1
        l1_output = F.relu(self.linear1(state))

        # Layer 1 to Layer 2
        l2_output = F.relu(self.linear2(l1_output))

        # Layer 2 to Action
        mean = self.action_mean(l2_output)
        std = self.action_std(l2_output)
        std = torch.clamp(std, min=LOG_SIG_MIN, max = LOG_SIG_MAX)
        std = std.exp()

        # Layer 2 to Value
        value_est = self.critic_head(l2_output)

        return value_est, mean, std

    def select_action(self,state):
        state = torch.from_numpy(state).float().unsqueeze(0)

        value_est, mean, std = self.forward(state)
        value_est = value_est.reshape(-1)

        # Make prob Normal dist
        dist = Normal(mean, std)

        action = dist.sample()
        action = torch.tanh(action)

        ln_prob = dist.log_prob(action)
        ln_prob = ln_prob.sum()

        self.saved_actions.append(SavedAction(ln_prob, value_est))

        action = action.numpy()

        return action[0]

    def compute_returns(self, gamma): # This is the error causing code
        """
        Calculate losses and do backprop
        """

        R = 0

        saved_actions = self.saved_actions

        policy_losses = []
        value_losses = []
        returns = []

        for r in self.rewards[::-1]:
            # Discount value
            R = r + gamma*R
            returns.insert(0,R)

        returns = torch.tensor(returns)
        returns = (returns - returns.mean())/(returns.std()+self.eps)

        for (log_prob, value), R in zip(saved_actions, returns):
            advantage = R - value.item()
            advantage = advantage.type(torch.FloatTensor)
            policy_losses.append(-log_prob*advantage)

            value_losses.append(F.mse_loss(value, torch.tensor([R])))

        self.optimizer.zero_grad()

        loss = torch.stack(policy_losses).sum() + torch.stack(value_losses).sum()

        loss = loss.type(torch.FloatTensor)
        loss.backward()
        self.optimizer.step()

        del self.rewards[:]
        del self.saved_actions[:]

env = gym.make("Pendulum-v0")

state_space = env.observation_space.shape[0]
action_space = env.action_space.shape[0]

# Train Expert AC

model = ActorCritic(state_space, action_space)

train = True
if train == True:

    # Main loop
    window = 50
    reward_history = []

    for ep in count():

        state = env.reset()

        ep_reward = 0

        for t in range(1,1000):

            if ep%50 == 0:
                env.render()

            action = model.select_action(state)

            state, reward, done, _ = env.step(action)

            model.rewards.append(reward)
            ep_reward += reward

            if done:
                break
        print(reward)
        model.compute_returns(0.99) # Error begins here
        reward_history.append(ep_reward)

        # Result information
        if ep % 50 == 0:
            mean = np.mean(reward_history[-window:])
            print(f"Episode: {ep} Last Reward: {ep_reward} Rolling Mean: {mean}")

        if np.mean(reward_history[-100:])>199:
            print(f"Environment solved at episode {ep}, average run length > 200")
            break

Complete error log, some elements redacted for privacy. Originally the loop actor critic and main loop were in separate files. Comments added to appropriate error causing lines.
Traceback (most recent call last):
  File "pendulum.py", line 59, in <module>
    model.compute_returns(0.99)
  File "/home/x/Software/git/x/x/solvers/actorcritic_cont.py", line 121, in compute_returns
    loss.backward()
  File "/home/x/.local/lib/python3.8/site-packages/torch/_tensor.py", line 255, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "/home/x/.local/lib/python3.8/site-packages/torch/autograd/__init__.py", line 147, in backward
    Variable._execution_engine.run_backward(
RuntimeError: Found dtype Double but expected Float

Output:
Answering here in case anyone has similar issues in the future.
The output of the reward in OpenAI Gym Pendulum-v0 is a double, so when you compute the return over the episode you need to change that to a float tensor.
I did this just by:
returns = torch.tensor(returns)
returns = (returns - returns.mean())/(returns.std()+self.eps)
returns = returns.type(torch.FloatTensor)


Instruction: How to serve a model in sagemaker?

Input: based on documentation provided here , https://sagemaker.readthedocs.io/en/stable/frameworks/pytorch/using_pytorch.html#model-directory-structure, the model file saved from training is model.pth. I also read that it can be .pt extension or even bin extension. I have seen a example of pytorch_model.bin, but when i tried to serve the model with the pytorch_model.bin, it warns me that .pt or .pth file needs to exist. has anyone run into this?

Output:
Interesting question.
I'm assuming you're trying to use the PyTorch container from SageMaker in what we call "script mode" - where you just provide the .py entrypoint.
Have you tried to define a model_fn() function, where you specify how to load your model? The documentation talks about this here.
More details:

Before a model can be served, it must be loaded. The SageMaker PyTorch model server loads your model by invoking a model_fn function that you must provide in your script when you are not using Elastic Inference.

import torch
import os
import YOUR_MODEL_DEFINITION

def model_fn(model_dir):
    model = YOUR_MODEL_DEFINITION()
    with open(os.path.join(model_dir, 'YOUR-MODEL-FILE-HERE'), 'rb') as f:
        model.load_state_dict(torch.load(f))
    return model

Let me know if this works!


Instruction: Preparing CSV file for neural network machine learning Python

Input:
I'm taking a course about machine learning in my undergrad studies and I have a problem where I don't know to load a CSV file  into Dataloader then test it, can someone guide me through the process?
you can download the CSV files from this link if you wish https://ufile.io/f/abdd9
Here is the code
import tensorflow as tf
from torch.utils.data import DataLoader
import numpy as np
import pandas as pd
import torch
import torchvision
import matplotlib.pyplot as plt
from time import time
from torchvision import datasets, transforms
from torch import nn, optim

train_data1 = pd.read_csv(("C:/Users/HP/OneDrive/سطح المكتب/KFUPM/TERM 212/EE485/Exp3/mnist_train.csv")
test_data1 = pd.read_csv("C:/Users/HP/OneDrive/سطح المكتب/KFUPM/TERM 212/EE485/Exp3/mnist_test.csv")
dtype = torch.float32
torch_tensor1 = torch.tensor(train_data1.values,dtype = dtype)
torch_tensor2 = torch.tensor(test_data1.values,dtype = dtype )
trainloader=DataLoader(torch_tensor1, batch_size=64, shuffle=True)
testloader =DataLoader(torch_tensor2, batch_size=64, shuffle=True)

then when i try to run this line of code i get an error
dataiter = iter(trainloader)
images, labels = dataiter.next()

print(images.shape)
print(labels.shape)

which is
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-234-afd0555e962e> in <module>
      1 dataiter = iter(trainloader)
----> 2 images, labels = dataiter.next()
      3
      4 print(images.shape)
      5 print(labels.shape)

ValueError: too many values to unpack (expected 2)

Output:
To do it properly with a Dataset and a Dataloader, you need to create a custom dataset:
import pandas as pd
from torch.utils.data import Dataset

class CustomMnistDataset(Dataset):
    def __init__(self, csv_file):
        data = pd.read_csv(csv_file)
        self.labels = np.array(data["label"])
        self.images = np.array(data.iloc[:, 1:])

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        return self.images[idx], self.labels[idx]

Then use it to create your dataloader:
from torch.utils.data import DataLoader

test_dataset = CustomMnistDataset("mnist_test.csv")
test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=True)
image_batch, label_batch = next(iter(test_dataloader))

This way you get a batch of 64 in the right Pytorch Tensor format for your training.
As I said in my comment, for MNIST it is an overkill as you can load it directly from Pytorch. You may need to flatten it though.
from torchvision import datasets

training_data = datasets.MNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor()
)

EDIT: If you want to use the dataset already provided in Pytorch in a flatten way, you can to this. Then the custom dataset is maybe simpler afterall.
from torchvision import datasets
from torchvision.transforms import ToTensor
import matplotlib.pyplot as plt

training_data = datasets.MNIST(
    root="data",
    train=True,
    download=True,
    transform=lambda x: torch.Tensor(np.array(x).reshape(len(np.array(x))**2))
)

train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)


Instruction: Generating histogram feature of 2D tensor from 3D Tensor feature set

Input:
I have a 3D tensor of dimensions (3,4 7) where each element in 2-dim(4) has 7 attributes.
What I want is to take the 4th attribute of all 4 elements and to calculate the histogram having 3 hist values and store those values only. And ending up with a 2D tensor of shape (3,4). I have a small toy example for the task that I am working on. My solution ends up with a Tensor which has shape (1,3). Any hint or guidance will be appreciated.
import torch
torch.manual_seed(1)
feature = torch.randint(1, 50, (3, 4,7))
feature.type(torch.FloatTensor)
attrbute_val = feature[:,:,3:4]
print(attrbute_val.shape)
print(attrbute_val)
histogram_feature = torch.histc(torch.tensor(attrbute_val,dtype=torch.float32), bins=3, min=1, max=50)
print("histogram_feature",histogram_feature)

Output:
import torch
torch.manual_seed(1)

bins = 3
feature = torch.randint(1, 50, (3, 4,7))

attrbute_val = feature[:,:,3].float() # read all 4 elements in the 2nd dimension
                                      # and the fourth element in the 3rd dimension.
final_tensor = torch.empty((bins,bins))

tuple_rows = torch.tensor_split(attrbute_val, 3, dim=0)

for i,row in enumerate(tuple_rows):
  final_tensor[i] = torch.histc(row, bins=bins, min=1, max=50)
  plt.bar(range(bins),final_tensor[i],align='center',color=['forestgreen'])
  plt.show()

#final_tensor = tensor([[3., 0., 1.],
#                       [4., 0., 0.],
#                       [0., 2., 2.]])


Instruction: How to create unnamed PyTorch parameters in state dict?

Input:
I am trying to load a model checkpoint (.ckpt file) for transfer learning. I do not have the model's source code, so I am trying to recreate it with PyTorch, like this:
import torch
import torch.nn as nn
import torch.nn.functional as F

class IngrDetNet(nn.Module):
    def __init__(self):
        super(IngrDetNet, self).__init__()
        self.fc1 = nn.Linear(n_ingr, 1024)
        self.fc2= nn.Linear(1024, 512)
        self.fc3 = nn.Linear(512, 256)
        self.fc4 = nn.Linear(256, n_ingr)

    def forward(self, x):
        x = self.fc1(x)
        x = F.leaky_relu(x, 0.2)
        x = self.fc2(x)
        x = F.leaky_relu(x, 0.2)
        x = self.fc3(x)
        x = F.leaky_relu(x, 0.2)
        x = self.fc4(x)

        ingrs = F.sigmoid(x)
        return ingrs

# Create a basic model instance
model = IngrDetNet()
model = torch.nn.DataParallel(model)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

The syntax is based on this PyTorch tutorial.
Then, I am trying to load in the checkpoint's state dict, based on this PyTorch tutorial:
model_path = Path('../Models/PITA/medr2idalle41.ckpt')
model.load_state_dict(torch.load(model_path, map_location=map_loc)['weights_id'])

But I get a key error mismatch:
Error(s) in loading state_dict for DataParallel:
    Missing key(s) in state_dict: "module.fc1.weight", "module.fc1.bias", "module.fc2.weight", "module.fc2.bias", "module.fc3.weight", "module.fc3.bias", "module.fc4.weight", "module.fc4.bias".
    Unexpected key(s) in state_dict: "module.model.0.weight", "module.model.0.bias", "module.model.2.weight", "module.model.2.bias", "module.model.4.weight", "module.model.4.bias".

The checkpoint state dict contains indexed keys such as module.model.0.weight, whereas my architecture contains named parameters such as module.fc1.weight. How do I generate layers in such a way that my parameters are not named but indexed?

Output:
The reason why there is a full mismatch of the keys is that you are using the nn.DataParallel module utility. This means it will wrap your original parent model under a wrapper "model" nn.Module. In other words:
>>> model = IngrDetNet()                  # model is a IngrDetNet
>>> model = torch.nn.DataParallel(model)  # model.model is a IngrDetNet

This in turn means your initialized model ends up with a prefixed "model." in its state dict keys.

You can fix this effect by changing the keys yourself before applying them on the model. A dict comprehension should do:
>>> state = torch.load(model_path, map_location=map_loc)
>>> state = {f'model.{k}': v for k, v in state['weights_id'].items()}

>>> model.load_state_dict(state)


Instruction: how to load two dataset images simultaneously for train two streams(Pytorch)

Input:
i need load identical two dataset suppose one dataset has RGB images and another dataset contain same image with different processed(grey images)  with same order same size,
datasetA=[1.jpg,2.jpg,..........n.jpg] // RGB
datasetA=[g1.jpg,g2.jpg,..........gn.jpg] //grey

so I need to feed the same order images to two independent networks using DataLoader with random_split, so how to use
rgb = datasets.ImageFolder(rgb images)
grey = datasets.ImageFolder(gray images)

train_data1, test_data = random_split(rgb, [train_data, test_data])
train_data2, test_data = random_split(grey, [train_data, test_data])

train_loader1 = DataLoader(train_data1, batch_size=batch_size, shuffle=True)
train_loader2 = DataLoader(train_data2, batch_size=batch_size, shuffle=True)

so need to load same order images touple like (1.jpg,g1.jpg) for train both network independantly
and how to use
trainiter1 = iter(train_loader1)
features, labels = next(trainiter)

please explain process

Output:
I think he easiest way to go about this is to construct a custom Dataset that handles both:
class JointImageDataset(torch.utils.data.Dataset):
  def __init__(self, args_rgb_dict, args_grey_dict):
    # construct the two individual datasets
    self.rgb_dataset = ImageFolder(**args_rgb_dict)
    self.grey_dataset = ImageFolder(**args_grey_dict)

  def __len__(self):
    return min(len(self.rgb_dataset), len(selg.grey_dataset))

  def __getitem__(self, index):
    rgb_x, rgb_y = self.rgb_dataset[index]
    grey_x, grey_y = self.grey_dataset[index]
    return rgb_x, grey_x, rgb_y, grey_y

Now you can construct a single DataLoader from the JoindImageDataset and iterate over the joint batches:
joint_data = JoindImageDataset(...)
train_loader = DataLoader(joint_data, batch_size=...)
for rgb_batch, grey_batch, rgb_ys, grey_ys in train_loader:
  # do your stuff here...


Instruction: when use conv and deconv, the out put shape does not math(The input image's weight is odd)

Input:
such as the input shape=[1,64,12,60,33]
when i use
nn.Conv3d(in_channels=128, out_channels=64, kernel_size=(3, 3, 3), stride=2, padding=1)

the out put shape =[1,64,6,30,17]
after that i want to let the output return to [1,64,12,60,33]
but when i use
 nn.ConvTranspose3d(in_channels=128, out_channels=64, kernel_size=(3, 3, 3), stride=2, padding=1,output_padding=1)

the out put become to [1, 64, 12, 60, 34] that is not i want.
how can i fixed this problem? i mean i want the nextwork doesn't matter the input's shape(of course i don't use dense layer, just conv and deconv)
for example:
input = torch.randn((1,64,12,60,33))
C3d=torch.nn.Conv3d(64,64,kernel_size=(3,3,3),stride=2 ,padding=1)
output_conv = C3d(input)#shape==[1,64,6,30,17]
de_C3d = torch.nn.ConvTranspose3d(64,64,(3,3,3),stride=2,padding=1)
output_deconv=de_C3d(out_conv) #shape = [1,64,11,59,33]

i just want the output_deconv.shape equal to input

Output:
If you're dealing with tensors of arbitrary shapes, this can be difficult. If they're fixed you can add ad hoc fixes which should solve your problem. One way is to utilise the fact that you can pass tuples to the arguments padding and output_padding, which will work in your case:
input = torch.randn((1,64,12,60,33))
C3d=torch.nn.Conv3d(64,64,kernel_size=(3,3,3),stride=2 ,padding=1)
output_conv = C3d(input) #shape==[1,64,6,30,17]
de_C3d = torch.nn.ConvTranspose3d(64,64,(3,3,3),stride=2,padding=1,output_padding=(1,1,0))
output_de=de_C3d(out_conv) #shape = [1,64,12,60,33]

You could also pad and then crop, which is commonly done in UNet architectures:
de_C3d = torch.nn.ConvTranspose3d(64,64,(3,3,3),stride=2,padding=0)
output_deconv=de_C3d(out_conv) #shape = [1,64,13,61,35]
output_deconv = output_deconv[:,:,:input.shape[2],:input.shape[3],:input.shape[4]]

I guess one way to fix this is to add different padding to the inputs depending on whether they're odd or even:
de_C3d = torch.nn.ConvTranspose3d(64,64,(3,3,3),stride=2,padding=1,
          output_padding=tuple([(i+1)%2 for i in input.shape[2:]]))
output_deconv=de_C3d(out_conv) #shape = [1,64,12,60,33]


Instruction: How to show wandb training progress from run folder

Input: After training neural networks with wandb as the logger, I received a link to show the training results and a folder named "run-...", I assume that is the logging of the training process. Now I don't have that link, how to show the wandb training process from run folder?

Output:
The run folder name is constructed as run-<datetime>-<id>.
You can find the logs on the UI platform as long as you haven't yet deleted it online. I'm not sure it is yet possible to resync the local copy to the cloud.
One way to find your run across projects is to go on your profile page: https://wandb.ai/<username> and type the run's id in the search bar.


Instruction: Select on second dimension on a 3D pytorch tensor with an array of indexes

Input:
I am kind of new with numpy and torch and I am struggling to understand what to me seems the most basic operations.
For instance, given this tensor:
A = tensor([[[6, 3, 8, 3],
         [1, 0, 9, 9]],

        [[4, 9, 4, 1],
         [8, 1, 3, 5]],

        [[9, 7, 5, 6],
         [3, 7, 8, 1]]])

And this other tensor:
B = tensor([1, 0, 1])

I would like to use B as indexes for A so that I get a 3 by 4 tensor that looks like this:
[[1, 0, 9, 9],
 [4, 9, 4, 1],
 [3, 7, 8, 1]]

Thanks!

Output:
Ok, my mistake was to assume this:
A[:, B]

is equal to this:
A[[0, 1, 2], B]

Or more generally the solution I wanted is:
A[range(B.shape[0]), B]


Instruction: FileNotFoundError: Entity folder does not exist! in Google Colab

Input:
Can anyone help me in sorting out this issue?
When I run these lines in Colab
:param files_name: containing training and validation samples list file.
:param boxes_and_transcripts_folder: gt or ocr result containing transcripts, boxes and box entity type (optional).
:param images_folder: whole images file folder
:param entities_folder: exactly entity type and entity value of documents, containing json format file
:param iob_tagging_type: 'box_level', 'document_level', 'box_and_within_box_level'
:param resized_image_size: resize whole image size, (w, h)
:param keep_ratio: TODO implement this parames
:param ignore_error:
:param training: True for train and validation mode, False for test mode. True will also load labels, and files_name and entities_file must be set.
'''
class PICKDataset(Dataset):
    def __init__(self, files_name: str = None,
                 boxes_and_transcripts_folder: str = 'boxes_and_transcripts',
                 images_folder: str = 'images',
                 entities_folder: str = 'entities',
                 iob_tagging_type: str = 'box_and_within_box_level',
                 resized_image_size: Tuple[int, int] = (480, 960),
                 keep_ratio: bool = True,
                 ignore_error: bool = False,
                 training: bool = True
                 ):

        super().__init__()
        self._image_ext = None
        self._ann_ext = None
        self.iob_tagging_type = iob_tagging_type
        self.keep_ratio = keep_ratio
        self.ignore_error = ignore_error
        self.training = training
        assert resized_image_size and len(resized_image_size) == 2, 'resized image size not be set.'
        self.resized_image_size = tuple(resized_image_size)  # (w, h)

        if self.training:  # used for train and validation mode
            self.files_name = Path(files_name)
            self.data_root = self.files_name.parent
            self.boxes_and_transcripts_folder: Path = self.data_root.joinpath(boxes_and_transcripts_folder)
            self.images_folder: Path = self.data_root.joinpath(images_folder)
            self.entities_folder: Path = self.data_root.joinpath(entities_folder)
            if self.iob_tagging_type != 'box_level':
                if not self.entities_folder.exists():
                    raise FileNotFoundError('Entity folder is not exist!')

I get this error
RuntimeError: module compiled against API version 0xe but this version of numpy is 0xd
[2022-02-26 13:17:11,933 - train - INFO] - Distributed GPU training model start...
[2022-02-26 13:17:11,933 - train - INFO] - [Process 306] Initializing process group with: {'MASTER_ADDR': '127.0.0.1', 'MASTER_PORT': '29500', 'RANK': '0', 'WORLD_SIZE': '1'}
[2022-02-26 13:17:11,934 - train - INFO] - [Process 306] world_size = 1, rank = 0, backend=nccl
Traceback (most recent call last):
  File "train.py", line 162, in <module>
    entry_point(config)
  File "train.py", line 126, in entry_point
    main(config, local_master, logger if local_master else None)
  File "train.py", line 34, in main
    train_dataset = config.init_obj('train_dataset', pick_dataset_module)
  File "/content/PICK-pytorch/parse_config.py", line 105, in init_obj
    return getattr(module, module_name)(*args, **module_args)
  File "/content/PICK-pytorch/data_utils/pick_dataset.py", line 66, in __init__
    raise FileNotFoundError('Entity folder is not exist!')
FileNotFoundError: Entity folder is not exist!
Traceback (most recent call last):
  File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/usr/local/lib/python3.7/dist-packages/torch/distributed/launch.py", line 263, in <module>
    main()
  File "/usr/local/lib/python3.7/dist-packages/torch/distributed/launch.py", line 259, in main
    cmd=cmd)
subprocess.CalledProcessError: Command '['/usr/bin/python3', '-u', 'train.py', '--local_rank=0', '-c', 'config.json', '-d', '0', '--local_world_size', '1']' returned non-zero exit status 1.

What I have tried
Changing inputfilepath in PICKDataset, remving lines which rise error.
Complete Notebook

Output: There is error in https://github.com/wenwenyu/PICK-pytorch/blob/master/config.json file. you have to change the path of data as per your working directory. Check line number 61 to 64 and 73 to 76.


Instruction: Cant get the right yolor pre trained weights in YOLOR

Input:
I'm training a custom dataset in yolor. I successfully run its once but after some time, I cant manage to do it very well.
The first error I noticed is in the training part:
Traceback (most recent call last):   File "train.py", line 537, in <module>
    train(hyp, opt, device, tb_writer, wandb)   File "train.py", line 80, in train
    ckpt = torch.load(weights, map_location=device)  # load checkpoint   File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 595, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)   File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 764, in _legacy_load
    magic_number = pickle_module.load(f, **pickle_load_args)
_pickle.UnpicklingError: invalid load key, '<'.

then i traced it and found that the pre trained weights didnt load correctly with this code:
%cd /content/yolor
!bash scripts/get_pretrain.sh

ann give me this error:
/content/yolor
awk: cannot open ./cookie (No such file or directory)

rm: cannot remove './cookie': No such file or directory

and that's the first and main thing I noticed comparing to what I've done, it should load the weight in there.
its just giving me a pre-trained files with some HTML code.
im using goolge colab btw

Output:
For me, the easiest way was to download data on my laptop, then upload them and replace current HTML weights with the correct ones.
You will find two weights link to google drive in the get_pretrain.sh file
yolor_p6.pt: https://drive.google.com/uc?export=download&id=1Tdn3yqpZ79X7R1Ql0zNlNScB1Dv9Fp76
yolor_w6.pt: https://drive.google.com/uc?export=download&id=1UflcHlN5ERPdhahMivQYCbWWw7d2wY7U
curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=1Tdn3yqpZ79X7R1Ql0zNlNScB1Dv9Fp76" > /dev/null
curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=awk '/download/ {print $NF}' ./cookie&id=1Tdn3yqpZ79X7R1Ql0zNlNScB1Dv9Fp76" -o yolor_p6.pt
rm ./cookie
curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=1UflcHlN5ERPdhahMivQYCbWWw7d2wY7U" > /dev/null
curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=awk '/download/ {print $NF}' ./cookie&id=1UflcHlN5ERPdhahMivQYCbWWw7d2wY7U" -o yolor_w6.pt
rm ./cookie
I am not familiar with colab, that's why I used this simple solution.


Instruction: How to convert a (M, L) tensor to (N, L) based on counts vector of size (N) where M is sum of counts, using aggregation by adding

Input:
So I have a 2D tensor A of shape (M, L) and I want to convert it into B, a (N, L) tensor.
I also have a counts tensor C (N) which has the counts of how many rows belong to which group such that sum(C) = M.
For example :
# shape = (6, 3)
A = torch.tensor([[1, 2, 3],
                  [4, 5, 6],
                  [1, 1, 1],
                  [5, 3, 1],
                  [5, 7, 1],
                  [2, 1, 3]])

# counts
C = torch.tensor([2, 1, 3])
# torch.sum(C) == A.shape[0]

# final tensor shape = (3, 3)
B = torch.tensor([[5, 7, 9],
                  [1, 1, 1],
                  [12, 11, 5]])

The aggregation of rows is done by element wise addition.
I tried to create a simple function for this as follows :
def convert_to_batch_results(self, results, counts):
  knowledge_cumsum = torch.cumsum(results, dim=0) # [M, L]
  inds = torch.cumsum(counts, dim=0) - 1 # [N]
  knowledge = knowledge_cumsum[inds] # [N, L]
  diff = torch.zeros(knowledge.shape) # [N, L]
  diff[1:] = knowledge[:-1]
  user_knowledge = torch.sub(knowledge, diff) # [N, L]
  return user_knowledge

This works when counts vector has all elements non-zero.
But if counts have some 0 elements the summation becomes wrong.
I case counts have 0s, I want the corresponding output rows in B to have value 0s.
This is what I changed :

def convert_to_batch_results(results, counts):
  knowledge_cumsum = torch.cumsum(results, dim=0) # [M, L]
  inds = torch.cumsum(counts, dim=0) - 1 # [N] # print(inds)
  knowledge = knowledge_cumsum[inds] # [N, L]
  knowledge = torch.mul(knowledge, torch.sign(counts.unsqueeze(1)))
  diff = torch.zeros(knowledge.shape).to('cpu') # [N, L]
  diff[1:] = knowledge[:-1]
  user_knowledge = torch.sub(knowledge, diff) # [N, L]
  user_knowledge = torch.mul(user_knowledge, torch.sign(counts.unsqueeze(1)))
  return user_knowledge

And ran it like so :
res = torch.ones(5, 3)
counts = torch.tensor([2, 0, 3])
print(convert_to_batch_results(res, counts))

# Output :
# tensor([[2., 2., 2.],
#        [-0., -0., -0.],
#        [5., 5., 5.]])

# Expected Output :
# tensor([[2., 2., 2.],
#        [-0., -0., -0.],
#        [3., 3., 3.]])

I have tried some other things but am unable to get the correct results in case of zeros present in counts. Please help me with the correct way to achieve the desired results.
Just to reiterate, counts can have zeros anywhere multiple times like [0, 3, 4], [0, 3, 0, 0, 1], [4, 0, 6, 0 1], etc.
Edit :
Adding the solution here by modifying my first approach and Shai's answer below.
The problem was with leading zeros in count which were becoming -1 and used as index into cumsum giving us the last row. Fixed this by using a mask.
def convert_to_batch_results(self, A, C):
  I = C.cumsum(dim=0)-1
  I = torch.mul(A.cumsum(dim=0)[I, :], (I != -1).int().unsqueeze(1)) # here
  I = torch.cat((torch.zeros_like(A[:1,:]), I), dim=0)
  return I.diff(dim=0)

Output:
You should use torch.cumsum to solve your problem:
Your output is simply the cumulative sum of rows up to the row induces of C. Taking the diff of the cumulative sum from the beginning of the tensor will give you the sum over the intervals you want:
B = torch.cat((torch.zeros_like(A[:1,:]), A.cumsum(dim=0)[C.cumsum(dim=0)-1, :]), dim=0).diff(dim=0)

Works with zeros in C "out-of-the-box".
We can break this one-liner to better understand what is going on:
A.cumsum(dim=0)  # cumulative sum of each column of A from the very beginning
C.cumsum(dim=0)  # (2, 3, 6) - At what rows we want to look at the cumulative sum: the second row will give us the first row of B. The third one would be the second row of B _but_ including the first one, etc.
diff(dim=0)  # subtrat the cumulative rows to get the intervals we want of A into B.

This trick is a simplified 1D version of integral images.


Instruction: Finding patterns in time series with PyTorch

Input:
I started PyTorch with image recognition. Now I want to test (very basically) with pure NumPy arrays. I struggle with getting the setup to work, so basically I have vectors with values between 0 and 1 (normalized curves). Those vectors are always of length 1500 and I want to find e.g. "high values at the beginning" or "sine wave-like function", "convex", "concave" etc. stuff like that, so just shapes of those curves.
My training set consists of many vectors with their classes; I have chosen 7 classes. The net should be trained to classify a vector into one or more of those 7 classes (not one hot).
I'm struggling with multiple issues, but first my very basic Net
class Net(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
        super(Net, self).__init__()
        self.hidden_dim = hidden_dim
        self.layer_dim = layer_dim
        self.rnn = nn.RNN(input_dim, hidden_dim, layer_dim)
        self.fc = nn.Linear(self.hidden_dim, output_dim)

    def forward(self, x):
        h0 = torch.zeros(self.layer_dim, x.size(1), self.hidden_dim).requires_grad_()
        out, h0 = self.rnn(x, h0.detach())
        out = out[:, -1, :]
        out = self.fc(out)
        return out

network = Net(1500, 70, 20, 7)
optimizer = optim.SGD(network.parameters(), lr=learning_rate, momentum=momentum)

This is just a copy-paste from an RNN demo. Here is my first issue. Is an RNN the right choice? It is a time series, but then again it is an image recognition problem when plotting the curve.
Now, this here is an attempt to batch the data. The data object contains all training curves together with the correct classifiers.
def train(epoch):
    network.train()
    network.float()
    batching = True
    index = 0
    # monitor the cummulative loss for an epoch
    cummloss = []
    # start batching some curves
    while batching:
        optimizer.zero_grad()

        # here I start clustering come curves to a batch and normalize the curves
        _input = []
        batch_size = min(len(data)-1, index+batch_size_train) - index
        for d in data[index:min(len(data)-1, index+batch_size_train)]:
            y = np.array(d['data']['y'], dtype='d')
            y = np.multiply(y, y.max())
            y = y[0:1500]
            y = np.pad(y, (0, max(1500-len(y), 0)), 'edge')
            if len(_input) == 0:
                _input = y
            else:
                _input = np.vstack((_input, y))
        input = torch.from_numpy(_input).float()
        input = torch.reshape(input, (1, batch_size, len(y)))
        target = np.zeros((1,7))
        # the correct classes have indizes, to I create a vector with 1 at the correct locations
        for _index in np.array(d['classifier']):
            target[0,_index-1] = 1
        target = torch.from_numpy(target)

        # get the result form the network
        output = network(input)
        # is this a good loss function?
        loss = F.l1_loss(output, target)
        loss.backward()
        cummloss.append(loss.item())
        optimizer.step()
        index = index + batch_size_train
        if index > len(data):
            print(np.mean(cummloss))
            batching = False

for e in range(1, n_epochs):
    print('Epoch: ' + str(e))
    train(0)

The problem I'm facing right now is, the loss doesn't change very little, even with hundreds of epochs.
Are there existing examples of this kind of problem? I didn't find any, just pure png/jpg image recognition. When I convert the curves to png then I have a little issue to train a net, I took densenet and it worked just fine but it seems to be super overkill for this simple task.

Output:
This is just a copy-paste from an RNN demo. Here is my first issue. Is an RNN the right choice?

In theory what model you choose does not matter as much as "How" you formulate your problem.
But in your case the most obvious limitation you're going to face is your sequence length: 1500. RNN store information across steps and typically runs into trouble over long sequence with vanishing or exploding gradient.
LSTM net have been developed to circumvent this limitations with memory cell, but even then in the case of long sequence it will still be limited by the amount of information stored in the cell.
You could try using a CNN network as well and think of it as an image.

Are there existing examples of this kind of problem?

I don't know but I might have some suggestions : If I understood your problem correctly, you're going from a (1500, 1) input to a (7,1) output, where 6 of the 7 positions are 0 except for the corresponding class where it's 1.
I don't see any activation function, usually when dealing with multi class you don't use the output of the dense layer to compute the loss you apply a normalizing function like softmax and then you can compute the loss.


Instruction: Failed to install PyTorch

Input:
I tried my best to install Pytorch but each and every time I failed to install it.

Conda version: 4.6.14

I have used Preview(Nightly) and LTS versions to install but for both of times I have faced the same error like Solving environment: | Killed .

Preview(Nightly) command: conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch-nightly

LTS command: conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch-lts

Faced error is given in the attached file, please check it.

Output:
have you tried installing pytorch into a new environment? problems usually arise when you try to install it into your base environment.
conda create -n (NameOfEnviroment) -c pytorch pytorch torchvision

conda update --all


Instruction: Using Focal Loss for imbalanced dataset in PyTorch

Input:
I found this implementation of focal loss in GitHub and I am using it for an imbalanced dataset binary classification problem.
# IMPLEMENTATION CREDIT: https://github.com/clcarwin/focal_loss_pytorch
    class FocalLoss(nn.Module):
    def __init__(self, gamma=0.5, alpha=None, size_average=True):
        super(FocalLoss, self).__init__()
        self.gamma = gamma
        self.alpha = alpha
        if isinstance(alpha,(float,int)): self.alpha = torch.Tensor([alpha,1-alpha])
        if isinstance(alpha,list): self.alpha = torch.Tensor(alpha)
        self.size_average = size_average

    def forward(self, input, target):
        if input.dim()>2:
            input = input.view(input.size(0),input.size(1),-1)  # N,C,H,W => N,C,H*W
            input = input.transpose(1,2)    # N,C,H*W => N,H*W,C
            input = input.contiguous().view(-1,input.size(2))   # N,H*W,C => N*H*W,C
        target = target.view(-1,1)

        logpt = F.log_softmax(input)
        logpt = logpt.gather(1,target)
        logpt = logpt.view(-1)
        pt = Variable(logpt.data.exp())

        if self.alpha is not None:
            if self.alpha.type()!=input.data.type():
                self.alpha = self.alpha.type_as(input.data)
            at = self.alpha.gather(0,target.data.view(-1))
            logpt = logpt * Variable(at)

        loss = -1 * (1-pt)**self.gamma * logpt
        if self.size_average: return loss.mean()
        else: return loss.sum()

also
gamma=args.gamma
alpha=args.alpha

criterion = FocalLoss(gamma, alpha)
m = nn.Sigmoid()

I use the criterion as follows in train phase:
for i_batch, sample_batched in enumerate(dataloader_train):
            #pdb.set_trace()
            feats = torch.stack(sample_batched['image'])
            labels = torch.as_tensor(sample_batched['label']).cuda()
            print('feats shape: ', feats.shape)
            print('labels shape: ', labels.shape)
            output = model(feats)
            loss = criterion(m(output[:,1]-output[:,0]), labels.float())

The error is:
train: True test: False
preparing datasets and dataloaders......
creating models......

=>Epoches 1, learning rate = 0.0010000, previous best = 0.0000
training...
feats shape:  torch.Size([64, 419, 512])
labels shape:  torch.Size([64])
main_classifier.py:86: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.
  logpt = F.log_softmax(input)
Traceback (most recent call last):
  File "main_classifier.py", line 346, in <module>
    loss = criterion(m(output[:,1]-output[:,0]), labels.float())
  File "/home/jalal/research/venv/dpcc/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "main_classifier.py", line 87, in forward
    logpt = logpt.gather(1,target)
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

How should I fix this error?
Is this implementation of FocalLoss correct?

Output:
Unlike BCEWithLogitLoss, inputting the same arguments as you would use for CrossEntropyLoss solved the problem:
#loss = criterion(m(output[:,1]-output[:,0]), labels.float())
loss = criterion(output, labels)

Credits to Piotr from NVidia


Instruction: Optimising model.parameters and custom learnable parameter together using torch.optim gives non-leaf tensor error

Input:
Framework: PyTorch
I am trying to optimise a custom nn.parameter(Temperature) used in softmax calculation along with the model parameters using a single Adam optimiser while model training. But doing so gives the following error:

ValueError: can't optimize a non-leaf Tensor

Here is my custom loss function:
class CrossEntropyLoss2d(torch.nn.Module):
    def __init__(self, weight=None):
        super().__init__()
        self.temperature = torch.nn.Parameter(torch.ones(1, requires_grad=True, device=device))
        self.loss = torch.nn.NLLLoss(weight)
        self.loss.to(device)

    def forward(self, outputs, targets):
        T_logits = self.temp_scale(outputs)
        return self.loss(torch.nn.functional.log_softmax(T_logits, dim=1), targets)

    def temp_scale(self, logits):
        temp = self.temperature.unsqueeze(1).expand(logits.size(1), logits.size(2), logits.size(3))
        return logits/temp
.
.
.
.
.
.

Here is the part of training code:
criterion = CrossEntropyLoss2d(weight)
params = list(model.parameters()) +list(criterion.temperature)
optimizer = Adam(params, 5e-4, (0.9, 0.999),  eps=1e-08, weight_decay=1e-4)

Error:
File "train_my_net_city.py", line 270, in train
optimizer = Adam(params, 5e-4, (0.9, 0.999),  eps=1e-08, weight_decay=1e-4)
File "/home/saquib/anaconda3/lib/python3.8/site-packages/torch/optim/adam.py", line 48, in __init__
super(Adam, self).__init__(params, defaults)
File "/home/saquib/anaconda3/lib/python3.8/site-packages/torch/optim/optimizer.py", line 54, in __init__
self.add_param_group(param_group)
File "/home/saquib/anaconda3/lib/python3.8/site-packages/torch/optim/optimizer.py", line 257, in add_param_group
raise ValueError("can't optimize a non-leaf Tensor")
ValueError: can't optimize a non-leaf Tensor

Checking the variable for leaf gives true:
print(criterion.temperature.is_leaf)
True

The error arises due to the criterion.temperature parameter and not due to model.parameters.

Output:
Got it working by doing so:
params = list(model.parameters())
params.append(criterion.temperature)


Instruction: Pytorch loss.backward() gives none grad for parameters of Rx, Ry Gate

Input:
I'm trying to train parameters params by performing linear Transformation on an input tensor x by matrix multiplying Rx to input followed by Ry matrix to their result. (each matrix Rx and Ry have a parameter params[i] each that define the matrix).
then I calculate loss by mse of y and the predicted output. when I do loss.backward()
Im getting params.grad  as None.
import torch

def get_device(gpu_no):
    if torch.cuda.is_available():
        return torch.device('cuda', gpu_no)
    else:
        return torch.device('cpu')

device = get_device(0)

params = torch.tensor(([[0.011], [0.012]]), requires_grad=True).to(device).to(torch.cfloat)

x_gate = torch.tensor([[1., 0.], [0., 1.]]).to(device)
y_gate = torch.tensor(([[0, -1j], [1j, 0]])).to(device)

def rx(theta):
    # co = torch.cos(theta / 2)
    # si = torch.sin(theta / 2)
    # Rx_gate = torch.stack([torch.cat([co, -si], dim=-1),
    #                       torch.cat([-si, co], dim=-1)], dim=-2).squeeze(0).to(device).to(torch.cfloat).requires_grad_()
    # Rx_gate = torch.exp(-1j * (theta / 2) * x_gate).to(device).to(torch.cfloat).requires_grad_()
    Rx_gate = torch.tensor(([[torch.cos(theta/2), -torch.sin(theta/2)],
                             [-torch.sin(theta/2), torch.cos(theta/2)]]), requires_grad=True).to(device).to(torch.cfloat)

    return Rx_gate

def ry(theta):
    # co = torch.cos(theta / 2)
    # si = torch.sin(theta / 2)
    # Ry_gate = torch.stack([torch.cat([co, -si]),
    #                             torch.cat([si, co])], dim=-2).squeeze(0).to(device).to(torch.cfloat).requires_grad_()
    # Ry_gate = torch.exp(-1j * (theta / 2) * y_gate).to(device).to(torch.cfloat).requires_grad_()
    Ry_gate = torch.tensor(([[torch.cos(theta / 2), -torch.sin(theta / 2)],
                        [torch.sin(theta / 2), torch.cos(theta / 2)]]), requires_grad=True).to(device).to(torch.cfloat)

    return Ry_gate

x = torch.tensor([1., 0.]).to(device).to(torch.cfloat)
y = torch.tensor([0., 1.]).to(device).to(torch.cfloat)

def pred(params):
    out = rx(params[0]) @ x
    out = ry(params[1]) @ out
    return out

print("params        :", params)
print("prediction    :", pred(params))

loss = torch.pow((y - pred(params)), 2).sum()
print("loss          :", loss)

loss.backward()
print("loss grad     :", loss.grad)
print("params grad   :", params.grad)

my output is
params        : tensor([[0.0110+0.j],
        [0.0120+0.j]], device='cuda:0', grad_fn=<ToCopyBackward0>)
prediction    : tensor([1.0000e+00+0.j, 5.0000e-04+0.j], device='cuda:0',
       grad_fn=<MvBackward0>)
loss          : tensor(1.9990+1.7485e-07j, device='cuda:0', grad_fn=<SumBackward0>)
loss grad     : None
params grad   : None

why grad is none even though params has grad_fn=<ToCopyBackward0>.
Also i get this warning:
 UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations. (Triggered internally at  aten\src\ATen/core/TensorBody.h:417.)

return self._grad

Output:
Good observation, you indeed have a correct backpropagation of the gradient through the gradient. So why are you getting none when accessing your parameter?

The reason why you can't access the gradient of this parameter is that only leaf tensors have their gradient cached in memory. Here, since params is a copy of a leaf-tensor (you called to twice on it which made that happen), it will not be considered a gradient of the computation graph.
In order to get access to the gradient of that parameter at runtime, you can force the engine to cache and make it accessible outside with a simple call to retain_grad as suggested by the warning message.
params.retain_grad()


Instruction: Log metrics with configuation in Pytorch Lightning using w&b

Input:
I am using PyTorch Lightning together with w&b and trying associate metrics with a finite set of configurations. In the LightningModule class I have defined the test_step as:
def test_step(self, batch, batch_idx):
  x, y_true, config_file =  batch
  y_pred = self.forward(x)
  accuracy = self.accuracy(y_pred, y_true)
  self.log("test/accuracy", accuracy)

Assuming (for simplicity) that the batch size is 1, this will log the accuracy for 1 sample and it will be displayed as a chart in the w&b dashboard.
I would like to associate this accuracy with some configuration of the experimental environment. This configuration might include BDP factor, bandwith delay, queue_size, location, etc. I don't want to plot the configurations I just want to be able to filter or group the accuracy by some configuration value.
The only solution I can come up with is to add these configurations as a querystring:
def test_step(self, batch, batch_idx):
  x, y_true, config_file =  batch
  # read values in config file
  # ...

  y_pred = self.forward(x)
  accuracy = self.accuracy(y_pred, y_true)
  self.log("test/BDP=2&delay=10ms&queue_size=10&topology=single/accuracy", accuracy)

Is there a better solution for this that integrates my desired functionality of being able to group and filter by values like BDP?

Output:
I work at W&B. You could log your config variables using wandb.config, like so:
wandb.config['my_variable'] = 123

And then you'll be able to filter your charts by whatever config you'd logged. Or am I missing something.
Possibly the save_hyperparameters call might even grab these config values automatically (from the WandbLogger docs here)
class LitModule(LightningModule):
    def __init__(self, *args, **kwarg):
        self.save_hyperparameters()


Instruction: How can I apply NMS (non-maximum suppression) on multiple images from a dataloader efficiently (PyTorch)?

Input:
I have the following function defined for non-maximum suppression (NMS) post processing on my predictions.
At the moment, it is defined for a single prediction or output:
from torchvision import transforms as torchtrans

def apply_nms(orig_prediction, iou_thresh=0.3):

    # torchvision returns the indices of the bboxes to keep
    keep = torchvision.ops.nms(orig_prediction['boxes'], orig_prediction['scores'], iou_thresh)

    final_prediction = orig_prediction
    final_prediction['boxes'] = final_prediction['boxes'][keep]
    final_prediction['scores'] = final_prediction['scores'][keep]
    final_prediction['labels'] = final_prediction['labels'][keep]

    return final_prediction

where I then apply it to a single image:
cpu_device = torch.device("cpu")

# pick one image from the test set
img, target = valid_dataset[3]
# put the model in evaluation mode
model.to(cpu_device)
model.eval()
with torch.no_grad():
    output = model([img])[0]

nms_prediction = apply_nms(output, iou_thresh=0.1)

However, I'm not sure how I can do this efficiently for a whole batch of images from a dataloader:
cpu_device = torch.device("cpu")
model.eval()
with torch.no_grad():
  for images, targets in valid_data_loader:
    images = list(img.to(device) for img in images)

    outputs = model(images)
    outputs = [{k: v.to(cpu_device)for k, v in t.items()} for t in outputs]
    #DO NMS POST PROCESSING HERE??

What would be the best approach? How can I apply the above defined function for multiple images? Would this be best done in another for loop?

Output:
Have a look at the Generic Trnasform paragraph in the torchivision doc page you can use torchvision.transform.Lambda or work with functional transforms.
Here is an example with Lambda
nms_transform = torchvision.transforms.Lambda(apply_nms)

Then, you can apply the transform with the transform parameter of your dataset (or you can create your custom dataset class, as well):
dset = MyDset(..., transform=torchvision.transforms.Compose([torchvision.transforms.ToTensor(), nms_transform()])


Instruction: optimizing multiple loss functions in pytorch

Input:
I am training a model with different outputs in PyTorch, and I have four different losses for positions (in meter), rotations (in degree), and velocity, and a boolean value of 0 or 1 that the model has to predict.
AFAIK, there are two ways to define a final loss function here:
one - the naive weighted sum of the losses
two - the defining coefficient for each loss to optimize the final loss.
So, My question is how is better to weigh these losses to obtain the final loss, correctly?

Output:
This is not a question about programming but instead about optimization in a multi-objective setup. The two options you've described come down to the same approach which is a linear combination of the loss term. However, keep in mind there are many other approaches out there with dynamic loss weighting, uncertainty weighting, etc... In practice, the most often used approach is the linear combination where each objective gets a weight that is determined via grid-search or random-search.
You can look up this survey on multi-task learning which showcases some approaches: Multi-Task Learning for Dense Prediction Tasks: A Survey, Vandenhende et al., T-PAMI'20.
This is an active line of research, as such, there is no definite answer to your question.


Instruction: Unable to install Torch with pipenv

Input:
I tried following this tutorial after not being able to lock with pipenv install torch I am using Linux Mint 20.3 una
pipenv install --extra-index-url https://download.pytorch.org/whl/cu113/ "torch==1.10.1+cu113"
caused this problem after a long 'installing torch...' stage:
Error:  An error occurred while installing torch==1.10.1+cu113!
Error text: Looking in indexes: https://download.pytorch.org/whl/cu113/, https://pypi.org/simple
Collecting torch==1.10.1+cu113
  Downloading https://download.pytorch.org/whl/cu113/torch-1.10.1%2Bcu113-cp39-cp39-linux_x86_64.whl (1821.5 MB)

✘ Installation Failed

This also did not work.
Installing torch==1.10.2+cu113...
Error:  An error occurred while installing torch==1.10.2+cu113!
Error text:
ERROR: Could not find a version that satisfies the requirement torch==1.10.2+cu113 (from versions: 1.7.1, 1.8.0, 1.8.1, 1.9.0, 1.9.1, 1.10.0, 1.10.1, 1.10.2)
ERROR: No matching distribution found for torch==1.10.2+cu113

Edit:
I have tried bu failed:
    pipenv install --extra-index-url https://download.pytorch.org/whl/ "torch==1.10.1+cu102"
    Installing torch==1.10.1+cu102...
    Adding torch to Pipfile's [packages]...
    ✔ Installation Succeeded
    Pipfile.lock (6516c9) out of date, updating to (2de599)...
    Locking [dev-packages] dependencies...
    Locking [packages] dependencies...
    Building requirements...
    Resolving dependencies...
    ✘ Locking Failed!

Output:
I solved it, I don't have those GPUs so I have to use the third command line of the tutorial
    pipenv install --extra-index-url https://download.pytorch.org/whl/ "torch==1.10.1+cpu"
    Installing torch==1.10.1+cpu...
    Adding torch to Pipfile's [packages]...
    ✔ Installation Succeeded
    Pipfile.lock (6516c9) out of date, updating to (3c44bd)...
    Locking [dev-packages] dependencies...
    Locking [packages] dependencies...
    Building requirements...
    Resolving dependencies...
    ✔ Success!
    Updated Pipfile.lock (3c44bd)!
    Installing dependencies from Pipfile.lock (3c44bd)...
         ▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ 0/0 — 00:


Instruction: What is cudaLaunchKernel in pytorch profiler output

Input:
I'm trying to profile my pytorch network to see what is the bottleneck. I noticed that there is an operation called cudaLaunchKernel which is taking up most of the time. This answer says that it is called for every operation done with cuda. If suppose I implement this network in C++ or any other language, would it be possible to reduce this time?
Basically, I'm asking if this overhead is because I've implemented my network in python or will this overhead be always there and impossible to optimize in any language?
Full profiler output:
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------
                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------
                                       cudaLaunchKernel        99.80%     933.739ms        99.80%     933.739ms      20.750ms       0.000us         0.00%       0.000us       0.000us            45
                                        model_inference         0.05%     453.000us       100.00%     935.567ms     935.567ms       0.000us         0.00%     195.000us     195.000us             1
                                aten::cudnn_convolution         0.04%     388.000us        99.84%     934.047ms     103.783ms     195.000us       100.00%     195.000us      21.667us             9
                                     aten::_convolution         0.01%     138.000us        99.88%     934.419ms     103.824ms       0.000us         0.00%     195.000us      21.667us             9
                                           aten::conv2d         0.01%     122.000us        99.89%     934.592ms     103.844ms       0.000us         0.00%     195.000us      21.667us             9
                                             aten::add_         0.01%     112.000us         0.02%     155.000us      17.222us       0.000us         0.00%       0.000us       0.000us             9
                               aten::upsample_nearest2d         0.01%      82.000us         0.01%     105.000us      26.250us       0.000us         0.00%       0.000us       0.000us             4
                                            aten::empty         0.01%      79.000us         0.01%      79.000us       3.292us       0.000us         0.00%       0.000us       0.000us            24
                                        aten::threshold         0.01%      74.000us         0.02%     149.000us      18.625us       0.000us         0.00%       0.000us       0.000us             8
                                             aten::_cat         0.01%      71.000us         0.01%     119.000us      29.750us       0.000us         0.00%       0.000us       0.000us             4
                                             aten::relu         0.01%      57.000us         0.02%     206.000us      25.750us       0.000us         0.00%       0.000us       0.000us             8
                                      aten::convolution         0.01%      51.000us        99.88%     934.470ms     103.830ms       0.000us         0.00%     195.000us      21.667us             9
                                             aten::view         0.01%      50.000us         0.01%      50.000us       5.556us       0.000us         0.00%       0.000us       0.000us             9
                                              aten::cat         0.00%      32.000us         0.02%     151.000us      37.750us       0.000us         0.00%       0.000us       0.000us             4
                                          aten::reshape         0.00%      29.000us         0.01%      79.000us       8.778us       0.000us         0.00%       0.000us       0.000us             9
                                          aten::resize_         0.00%      25.000us         0.00%      25.000us       0.962us       0.000us         0.00%       0.000us       0.000us            26
                                             aten::rsub         0.00%      21.000us         0.00%      33.000us      33.000us       0.000us         0.00%       0.000us       0.000us             1
                                              aten::mul         0.00%      17.000us         0.00%      27.000us      27.000us       0.000us         0.00%       0.000us       0.000us             1
                                            aten::zeros         0.00%      13.000us         0.00%      16.000us      16.000us       0.000us         0.00%       0.000us       0.000us             1
                                        cudaEventRecord         0.00%      12.000us         0.00%      12.000us       1.333us       0.000us         0.00%       0.000us       0.000us             9
                                        cudaBindTexture         0.00%      11.000us         0.00%      11.000us       2.750us       0.000us         0.00%       0.000us       0.000us             4
                                    aten::empty_strided         0.00%       6.000us         0.00%       6.000us       6.000us       0.000us         0.00%       0.000us       0.000us             1
                                            aten::zero_         0.00%       1.000us         0.00%       1.000us       1.000us       0.000us         0.00%       0.000us       0.000us             1
cudnn::maxwell::gemm::computeOffsetsKernel(cudnn::ma...         0.00%       0.000us         0.00%       0.000us       0.000us     195.000us       100.00%     195.000us     195.000us             1
                                      cudaUnbindTexture         0.00%       0.000us         0.00%       0.000us       0.000us       0.000us         0.00%       0.000us       0.000us             4
-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------
Self CPU time total: 935.583ms
Self CUDA time total: 195.000us

PS: Some configs
Python version: 3.8.8
PyTorch version: 1.8.1
cudatoolkit version: 10.2.89
cuda version (as given by nvidia-smi): 11.4
CPU specs: intel core i7 10700 @ 2.90GHz 16 cores
GPU specs: NVIDIA GM204GL [Quadro M4000]
RAM: 64GB
GPU RAM: 8GB
OS: 64-bit Ubuntu 20.04.3
PPS: I'm not looking for ways to speed up my code. I want to know if it is possible to speed it up by coding it in a different language like cpp or directly in cuda. (Like suppose if all my data is already on GPU, and I've written my code in cuda language itself, would it run in 195us?)

Output:
According to CUDA docs, cudaLaunchKernel is called to launch a device function, which, in short, is code that is run on a GPU device.
The profiler, therefore, states that a lot of computation is run on the GPU (as you probably expected) and this requires the data structures to be transferred on the device. This may be the source of the bottleneck.
I don't usually develop in CUDA, but perhaps you can speed up the process by coding larger kernels with more operation in CUDA and less CPU/GPU transferrals.
Have a look at this tutorial for more details.


Instruction: Pytorch Forecasting vs Darts, experiences welcome

Input:
I was wondering which package to use between pytorch forecasting (https://pytorch-forecasting.readthedocs.io/en/stable/) or darts (https://unit8co.github.io/darts/). I have been trying both, it looks like darts is more sklearn-like in its writing and style and pytorch forescasting uses different data classes.
Any comment comparing the two would be welcome.
I don't know if some of you might have carried out a performance comparison between both libraries.
Thanks in advance!

Output: I think one of the biggest advantage of darts is its Timeseries Object which is very pandas-like and very intuitive when you are familiar with sklearn. However, I also do see the advantage that pytorch-forecasting dealt with categorical data "better" (easier) and it takes a steeper learning curve to understand pytorch-forecasting. I would say pytorch-forecasting sometimes outperform darts using the same model.


Instruction: RuntimeError: mat1 and mat2 shapes cannot be multiplied (4000x20 and 200x441)

Input:
The architecture of the decoder of my variational autoencoder is given in the snippet below
class ConvolutionalVAE(nn.Module):

    def __init__(self, nchannel, base_channels, z_dim, hidden_dim,  device, img_width, batch_size):
        super(ConvolutionalVAE, self).__init__()

        self.nchannel   = nchannel
        self.base_channels = base_channels
        self.z_dim      = z_dim
        self.hidden_dim = hidden_dim
        self.device     = device
        self.img_width  = img_width
        self.batch_size = batch_size
        self.enc_kernel = 4
        self.enc_stride = 2
        self._to_linear = None
        ########################
        # ENCODER-CONVOLUTION LAYERS
        self.conv0       = nn.Conv2d(nchannel, base_channels, self.enc_kernel, stride=self.enc_stride)
        self.bn2d_0      = nn.BatchNorm2d(self.base_channels)
        self.LeakyReLU_0 = nn.LeakyReLU(0.2)
        out_width        = np.floor((self.img_width - self.enc_kernel) / self.enc_stride + 1)
        self.conv1       = nn.Conv2d(base_channels, base_channels*2, self.enc_kernel, stride=self.enc_stride)
        self.bn2d_1      = nn.BatchNorm2d(base_channels*2)
        self.LeakyReLU_1 = nn.LeakyReLU(0.2)
        out_width        = np.floor((out_width - self.enc_kernel) / self.enc_stride + 1)
        self.conv2       = nn.Conv2d(base_channels*2, base_channels*4, self.enc_kernel, stride=self.enc_stride)
        self.bn2d_2      = nn.BatchNorm2d(base_channels*4)
        self.LeakyReLU_2 = nn.LeakyReLU(0.2)
        out_width        = np.floor((out_width - self.enc_kernel) / self.enc_stride + 1)
        self.conv3       = nn.Conv2d(base_channels*4, base_channels*8, self.enc_kernel, stride=self.enc_stride)
        self.bn2d_3      = nn.BatchNorm2d(base_channels*8)
        self.LeakyReLU_3 = nn.LeakyReLU(0.2)
        out_width        = int(np.floor((out_width - self.enc_kernel) / self.enc_stride + 1))
        ########################
        #ENCODER-USING FULLY CONNECTED LAYERS
        #THE LATENT SPACE (Z)
        self.flatten     = nn.Flatten()
        self.fc0         = nn.Linear((out_width**2) * base_channels * 8, base_channels*8*4*4, bias=False)
        self.bn1d        = nn.BatchNorm1d(base_channels*8*4*4)
        self.fc1         = nn.Linear(base_channels*8*4*4, hidden_dim, bias=False)
        self.bn1d_1      = nn.BatchNorm1d(hidden_dim)
        # mean of z

        self.fc2         = nn.Linear(hidden_dim, z_dim, bias=False)
        self.bn1d_2      = nn.BatchNorm1d(z_dim)
        # variance of z

        self.fc3         = nn.Linear(hidden_dim, z_dim, bias=False)
        self.bn1d_3      = nn.BatchNorm1d(z_dim)
        ########################
        # DECODER:
        #  P(X|Z)
        conv2d_transpose_kernels, conv2d_transpose_input_width = self.determine_decoder_params(self.z_dim, self.img_width)
        self.conv2d_transpose_input_width = conv2d_transpose_input_width
        self.px_z_fc_0   = nn.Linear(self.z_dim, conv2d_transpose_input_width ** 2)
        self.px_z_bn1d_0 = nn.BatchNorm1d(conv2d_transpose_input_width ** 2)
        self.px_z_fc_1   = nn.Linear(conv2d_transpose_input_width ** 2, conv2d_transpose_input_width ** 2)
        #self.unflatten = nn.Unflatten(1, (1, conv2d_transpose_input_width, conv2d_transpose_input_width))
        self.conv2d_transpose_input_width = conv2d_transpose_input_width
        self.px_z_conv_transpose2d = nn.ModuleList()
        self.px_z_bn2d   = nn.ModuleList()
        self.n_conv2d_transpose = len(conv2d_transpose_kernels)
        self.px_z_conv_transpose2d.append(nn.ConvTranspose2d(1, self.base_channels * (self.n_conv2d_transpose - 1),
                                                             kernel_size=conv2d_transpose_kernels[0], stride=2))
        self.px_z_bn2d.append(nn.BatchNorm2d(self.base_channels * (self.n_conv2d_transpose - 1)))
        self.px_z_LeakyReLU = nn.ModuleList()
        self.px_z_LeakyReLU.append(nn.LeakyReLU(0.2))

        for i in range(1, self.n_conv2d_transpose - 1):
            self.px_z_conv_transpose2d.append(nn.ConvTranspose2d(self.base_channels * (self.n_conv2d_transpose - i),
                                                                 self.base_channels*(self.n_conv2d_transpose - i - 1),
                                                                 kernel_size=conv2d_transpose_kernels[i], stride=2))
            self.px_z_bn2d.append(nn.BatchNorm2d(self.base_channels * (self.n_conv2d_transpose - i - 1)))
            self.px_z_LeakyReLU.append(nn.LeakyReLU(0.2))
        self.px_z_conv_transpose2d.append(nn.ConvTranspose2d(self.base_channels, self.nchannel,
                                                             kernel_size=conv2d_transpose_kernels[-1], stride=2))
                self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
        self.to(device=self.device)

    def decoder(self, z_input):
        #Generate X: P(X|Z)
        h  = F.relu(self.px_z_bn1d_0(self.px_z_fc_0(z_input)))
        flattened_h = self.px_z_fc_1(h)
        h = flattened_h.view(flattened_h.size()[0], 1, self.conv2d_transpose_input_width, self.conv2d_transpose_input_width)
        for i in range(self.n_conv2d_transpose - 1):
            h = self.px_z_LeakyReLU[i](self.px_z_bn2d[i](self.px_z_conv_transpose2d[i](h)))
        x_recons_mean_flat = torch.sigmoid(self.px_z_conv_transpose2d[self.n_conv2d_transpose - 1](h))
        return x_recons_mean_flat

running my code to reconstruct the images:
all_z = []
for d in range(self.z_dim):
   temp_z = torch.cat( [self.z_sample_list[k][:, d].unsqueeze(1) for k in range(self.K)], dim=1)
   print(f'size of each z component dimension: {temp_z.size()}')
   all_z.append(torch.mm(temp_z.transpose(1, 0), components).unsqueeze(1))
out       = torch.cat( all_z,1)
x_samples       = self.decoder(out)

I got this error message:
size of z dimension: 200
size of each z component dimension: torch.Size([50, 20])
size of all z component dimension: torch.Size([20, 200, 20])
x_samples = self.decoder(out)
File "VAE.py", line 241, in decoder
h  = F.relu(self.px_z_bn1d_0(self.px_z_fc_0(z_input)))
File "/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/anaconda3/lib/python3.8/site-packages/torch/nn/modules/linear.py", line 96, in forward
return F.linear(input, self.weight, self.bias)
File "/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py", line 1847, in linear
return torch._C._nn.linear(input, weight, bias)
RuntimeError: mat1 and mat2 shapes cannot be multiplied (4000x20 and 200x441)

Update
I changed my code slightly to this
all_z = []
for d in range(self.z_dim):
    temp_z = torch.cat( [self.z_sample_list[k][:, d].unsqueeze(1) for k in range(self.K)], dim=1)
    all_z.append(torch.mm(temp_z.transpose(1, 0), components).unsqueeze(1))
out       = torch.cat( all_z,1)
print(f'size of all z component dimension: {out.size()}')
out = F.pad(input=out, pad=(1, 0, 0,0, 0, 1), mode='constant', value=0)
print(f'new size of all z component dimension after padding: {out.size()}')
out = rearrange(out, 'd0 d1 d2 -> d1 (d0 d2)')
x_samples       = self.decoder(out)

Now the new error is
x_samples       = self.decoder(out)
File "VAE.py", line 243, in decoder
h  = F.relu(self.px_z_bn1d_0(self.px_z_fc_0(z_input)))
File "/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "/anaconda3/lib/python3.8/site-packages/torch/nn/modules/linear.py", line 96, in forward
return F.linear(input, self.weight, self.bias)
File "/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py", line 1847, in linear
return torch._C._nn.linear(input, weight, bias)
RuntimeError: mat1 and mat2 shapes cannot be multiplied (200x441 and 200x441)

Any suggestion to fix this error?

Output:
Matrix multiplication requires the 2 inner dimensions to be the same. You are getting the error: RuntimeError: mat1 and mat2 shapes cannot be multiplied (200x441 and 200x441) because your inner dimensions don't line up.
for example:
shape(200, 441) * shape(441, 200) # works
shape(441, 200) * shape(200, 441) # works
shape(200, 441) * shape(200, 441) # doesn't work, this is why you are getting your error

# in general
shape(x, y) * shape(y, z) # works

To make the inner dimensions match, just take the transpose of one or the other:
shape(200, 441) * shape(200, 441).T # works
# or
shape(200, 441).T * shape(200, 441) # works

# since the transpose works by swapping the dimensions:
shape(200, 441).T = shape(441, 200)


Instruction: How do I compute batched sample covariance in PyTorch?

Input:
Say I have data, a batched tensor of collections of data points of size (B, N, D) where B is my batch size, N is the number of data samples in each collection, and D is the length of my data vectors. I want to compute the sample mean and covariance for each collection of data points, but do it in batch.
To compute the mean I can do data.mean(dim=1) and I get a tensor of size (B, D) representing the mean of each collection. I assumed I'd be able to do a similar thing with torch.cov but it does not offer the ability to do it in batch. Is there another way to achieve this? I'm expecting to get a batch of covariance matrices of shape (B, D, D).

Output:
This does the trick:
def batch_cov(points):
    B, N, D = points.size()
    mean = points.mean(dim=1).unsqueeze(1)
    diffs = (points - mean).reshape(B * N, D)
    prods = torch.bmm(diffs.unsqueeze(2), diffs.unsqueeze(1)).reshape(B, N, D, D)
    bcov = prods.sum(dim=1) / (N - 1)  # Unbiased estimate
    return bcov  # (B, D, D)

Here is a script to test that it's computing the same thing that the non-batched PyTorch version computes:
import time
import torch

B = 10000
N = 50
D = 2
points = torch.randn(B, N, D)
start = time.time()
my_covs = batch_cov(points)
print("My time:   ", time.time() - start)

start = time.time()
torch_covs = torch.zeros_like(my_covs)
for i, batch in enumerate(points):
    torch_covs[i] = batch.T.cov()

print("Torch time:", time.time() - start)
print("Same?", torch.allclose(my_covs, torch_covs, atol=1e-7))

Which gives me:
My time:    0.00251793861318916016
Torch time: 0.2459864616394043
Same? True

I can't claim mine will be inherently faster than iteratively computing them, it seems as D gets bigger mine will slow down much more, so there's probably a nicer way to scale with data dimension size.


Instruction: Pytorch netwrok with variable number of hidden layers

Input:
I want to create a class that creates a simple network with X fully connected layers, where X is an input given by the user. I tried this using the setattr/getattr but for some reason is not working.
class MLP(nn.Module):
      def __init__(self,in_size, out_size,n_layers, hidden_size):
        super(MLP,self).__init__()
        self.n_layers=n_layers

        for i in range(n_layers):
            if i==0:
                layer_in_size = in_size
            else:
                layer_in_size = hidden_size

            if i==(n_layers-1):
                layer_out_size = out_size
            else:
                layer_out_size = hidden_size

            setattr(self,'dense_{}'.format(i), nn.Linear(layer_in_size,layer_out_size))

        def forward(self,x):
            out = x
            for i in range(self.n_layers):
                if i==(self.n_layers-1):
                    out = getattr(self,'dense_{}'.format(i),out)
                else:
                    out = F.relu(getattr(self,'dense_{}'.format(i),out))

            return out

This is the error im getting when trying a forward pass with the net:
enter image description here
Some insights of what's the issue will be helpful.

Output: This seems like a problem with forward implementation with the mod2 function. Try the pytorch functions (torch.fmod and torch.remainder) or if you don't need the backprop capabilities try to do .detach() before the mod2 function.


Instruction: How to handle that error in pytorch: expected Long but found double

Input:
I'm given a function that is supposed to calculate the square-root of a matrix
import torch
from torch.autograd import Function

class MatrixSquareRoot(Function):
  """Square root of a positive definite matrix.
  NOTE: matrix square root is not differentiable for matrices with
  zero eigenvalues.
  See Lin, Tsung-Yu, and Subhransu Maji.
  "Improved Bilinear Pooling with CNNs." BMVC 17
  """
  @staticmethod
  def forward(ctx, input):
    dim = input.shape[0]
    norm = torch.norm(input.double())
    Y = input/norm
    I = torch.eye(dim,dim,device=input.device).type(input.dtype)
    Z = torch.eye(dim,dim,device=input.device).type(input.dtype)
    for i in range(15):
      T = 0.5*(3.0*I - Z.mm(Y))
      Y = Y.mm(T)
      Z = T.mm(Z)
      sqrtm = Y*torch.sqrt(norm)
      #ctx.mark_dirty(Y,I,Z)
      ctx.save_for_backward(sqrtm)
    return sqrtm #, I, Y, Z

  @staticmethod
  def backward(ctx, grad_output):
    grad_input = None
    sqrtm, = ctx.saved_tensors
    dim = sqrtm.shape[0]
    norm = torch.norm(sqrtm)
    A = sqrtm/norm
    I = torch.eye(dim, dim, device=sqrtm.device).type(sqrtm.dtype)
    Q = grad_output/norm
    for i in range(15):
      Q = 0.5*(Q.mm(3.0*I-A.mm(A))-A.t().mm(A.t().mm(Q)-Q.mm(A)))
      A = 0.5*A.mm(3.0*I-A.mm(A))
      grad_input = 0.5*Q
    return grad_input

sqrtm = MatrixSquareRoot.apply # call: sqrtm(tensor of size d x d)

But when I'm trying to apply it to a matrix that has a square root, I'm getting that error:
>>> x = torch.tensor([[1,-12],[0,4]])
>>> sqrtm(x)

---------------------------------------------------------------------------
<ipython-input-28-9310ac844935> in forward(ctx, input)
     15     Z = torch.eye(dim,dim,device=input.device).type(input.dtype)
     16     for i in range(15):
---> 17       T = 0.5*(3.0*I - Z.mm(Y))
     18       Y = Y.mm(T)
     19       Z = T.mm(Z)

RuntimeError: expected scalar type Long but found Double

I also tried to convert to Long by calling sqrtm(x.type(torch.LongTensor)) instead but it does produce the same error.

Output:
just change the tensor type as follows
x = torch.tensor([[1,-12],[0,4]],dtype=torch.float)
sqrtm(x)


Instruction: Segfault while importing torchvision.transforms

Input:
I'm getting a segfault in python during imports.
This code:
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
print("was there")
from torchvision import transforms
print("didn't get there")
from torchvision import datasets
from torchvision import models

returns this:
$ python3 -u classifier.py
was there
Erreur de segmentation (core dumped)

So torchvision.transforms seems to be responsible. I've tried switching the lines, and torchvision.models fails too.
I've also tried importing torchvision.transforms on it's own and there were no problems. What could possibly cause this?
Edit:
I'm working on Ubuntu 20.04.4 and installed torchvision through pip.

Output:
So I moved the torchvision.transforms import to above the matplotlib.pyplot one, and somehow neither torchvision.transforms nor torchvision.models cause a segfault anymore. It still caused a segfault with torchvision.transforms right after matplotlib.pyplot.
Here is what the final code looks like:
import os
from torchvision import transforms
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets
from torchvision import models

At least my code works, but I feel like there must be an underlying problem that this doesn't adress...


Instruction: Difference between model.parameters and model.parameters(), pytorch

Input:
I have read through the documentation and I don't really understand the explanation. Here is the explanation I got from the documentation Returns an iterator over module parameters. Why does model.parameters() return the file location e.g <generator object Module.parameters at 0x7f1b90c29ad0>. model.parameters will give me an output of
<bound method Module.parameters of ResNet9(
  (conv1): Sequential(
    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
  )
  (conv2): Sequential(
    (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
    (3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  ).....

Output:
model.parameters()
It's simply because it returns an iterator object, not a list or something like that. But it behaves quite similar to a list. You can iterate over it eg with
[x for x in model.parameters()]
Or you can convert it to a list
[list(model.parameters())]
Iterators have some advantages over lists. Eg they are not "calculated" when they are created, which can improve performance.
For more on iterators just google for "python iterators", you'll find plenty of information. Eg w3schools.com is a good source.
model.parameters
The output model.parameters consists of two parts.
The first part bound method Module.parameters of tells you that you are referencing the method Module.parameters.
The second part tells you more about the object containing the referenced method. It' s the "object description" of your model variable. It's the same as print(model)
more on python references
model.parameter is just a reference to the parameter function, it's not executing the function. model.parameter() instead is executing it.
Maybe it gets more clear with a simple example.
print("Hello world")
>>> Hello world
print
>>> <function print>

abc = print

abc("Hello world")
>>> Hello world
abc
>>> <function print>

As you see abc behave exactly the same as print because i assigned the reference of print to abc.
If I would have executed the function instead, eg abc = print("Hello world"), abc would contain the string Hello world and not the function reference to print.


Instruction: Dict support in PyTorch

Input:
Does PyTorch support dict-like objects, through which we can backpropagate gradients, like Tensors in PyTorch?
My goal is to compute gradients with respect to a few (1%) elements of a large matrix. But if I use PyTorch's standard Tensors to store the matrix, I need to keep the whole matrix in my GPU, which causes problems due to limited GPU memory available during training. So I was thinking whether I could store the matrix as a dict instead, indexing only the relevant elements of the matrix, and computing gradients and backpropagating w.r.t those select elements only.
So far, I have tried using Tensors only, but it's causing memory issues for the above reasons. So I searched extensively for alternate options like dicts in PyTorch but couldn't find any such information on Google.

Output:
It sounds like you want your parameter to be a torch.sparse tensor.
This interface allows you to have tensors that are mostly zeros, with only a few non-zero elements in known locations. Sparse tensors should allow you to significantly reduce the memory footprint of your model.
Note that this interface is still "under construction": not all operations are supported for sparse tensors. However, it is being constantly improving.


Instruction: Does Fine-tunning Bert Model in multiple times with different dataset make it more accuracy?

Input:
i'm totally new in NLP and Bert Model.
What im trying to do right now is Sentiment Analysis on Twitter Trending Hashtag ("neg", "neu", "pos") by using DistilBert Model, but the accurazcy was about 50% ( I tried w Label data taken from Kaggle).
So here is my idea:
(1) First, I will Fine-tunning Distilbertmodel (Model 1) with IMDB dataset,
(2) After that since i've got some data took from Twitter post,  i will sentiment analysis them my Model 1 and get Result 2.
(3) Then I will refine-tunning Model 1 with the Result 2 and expecting to have Model (3).
Im not really sure this process has any meaning to make the model more accuracy or not.
Thanks for reading my post.

Output:
If you want to fine-tune a sentiment classification head of BERT for classifying tweets, then I'd recommend a different strategy:

IMDB dataset is a different kind of sentiment - the ratings do not really correspond with short post sentiment, unless you want to focus on tweets regarding movies.

using classifier output as input for further training of that classifier is not really a good approach, because, if the classifier made many mistakes while classifying, these will be reflected in the training, and so the errors will deapen. This is basically creating endogenous labels, which will not really improve your real-world classification.

You should consider other ways of obtaining labelled training data. There are a few good examples for twitter:

Twitter datasets on Kaggle - there are plenty of datasets available containing millions of various tweets. Some of those even contain sentiment labels (usually inferred from emoticons, as these were proven to be more accurate than words in predicting sentiment - for explanation see e.g. Frasincar 2013). So that's probably where you should look.

Stocktwits (if youre interested in financial sentiments)- contain posts that authors can label for sentiments, thus are a perfect way of mining labelled data, if stocks/cryptos is what you're looking for.

Another thing is picking a model that's better for your language, I'd recommend this one. It has been pretrained on 80M tweets, so should provide strong improvements. I believe it even contains a sentiment classification head that you can use.
Roberta Twitter Base
Check out the website for that and guidance for loading the model in your code - it's very easy, just use the following code (this is for sentiment classification):
MODEL = "cardiffnlp/twitter-roberta-base-sentiment"

tokenizer = AutoTokenizer.from_pretrained(MODEL)

model = AutoModelForSequenceClassification.from_pretrained(MODEL)

Another benefit of this model is that it has been pretrained from scratch with a vocabulary that contains emojis, meaning it has a deep understanding of them, their typical contexts and co-occurences. This can greatly benefit a social media classification, as many researchers would agree that emojis/emoticons are better predictors of sentiment than normal words.


Instruction: Python Pytorch Multiprocessing Pycharm PicklingError: Can't pickle : attribute lookup train on __main__ failed

Input:
This error happens when running multiprocessing (using spawn method) in Python or Pytorch (torch.multiprocessing) using Pycharm 2021.2.3.
The function train is defined at the top level of the module, so it should be pickable. However, the error says that it cannot be pickled.
A simple code may look like this:
def train(gpu):
    print(f'hello {gpu}!')

if __name__ == '__main__':
    mp.spawn(train, nprocs=2)

Output:
Seems like this is a bug in Pycharm 2021.2.3 and happens when Run with Python Console is checked in run configurations. This bug is being tracked at https://youtrack.jetbrains.com/issue/PY-50116
This can be resolved using the following two options (until the bug is resolved):

Uncheck Run with Python Console
Downgrade to Pycharm 2021.1.3


Instruction: Simple MultiGPU during inference with huggingface

Input:
I have two GPU.
How can I use them for inference with a huggingface pipeline?
Huggingface documentation seems to say that we can easily use the DataParallel class with a huggingface model, but I've not seen any example.
For example with pytorch, it's very easy to just do the following :
net = torch.nn.DataParallel(model, device_ids=[0, 1, 2])
output = net(input_var)  # input_var can be on any device, including CPU

Is there an equivalent with huggingface ?

Output:
I found it's not possible with the pipelines, so:
two ways :

Do it with the Trainer object in huggingface , which also supports inferences, but it's not optimal.
Use Queues from the multiprocessing standard library, but this creates a lot of boiler plate code


Instruction: How does Pytorch no_grad function for a = a - b and a -= b type of operation?

Input:
import torch

def model(x, W, b):
    return x@W + b

def mse(t1, t2):
    diff = t1 - t2
    return torch.sum(diff * diff) / diff.numel()

inputs = torch.rand(2, 3, requires_grad=True)
targets = torch.rand( 2,2, requires_grad=True)

W = torch.rand(3, 2, requires_grad=True)
b = torch.rand(2, requires_grad=True)

pred = model(inputs, W, b)

loss = mse(pred, targets)
loss.backward()

print(W.grad)
print(b.grad)
with torch.no_grad():
    W -= W.grad * 1e-5
    b -= b.grad * 1e-5
print(W.grad)
print(b.grad)

For the above example, the output of the last 2 print statements is the same as that of the first two print statements.
But, for the code snippet below, the 2 last print statements give result as None.
I can't understand why it is so?
print(W.grad)
print(b.grad)
with torch.no_grad():
    W = W - W.grad * 1e-5
    b = b -  b.grad * 1e-5
print(W.grad)
print(b.grad)

Output:
Keep in mind in both scenarios you are under the torch.no_grad context manager which by effect disables gradient computation.
On one hand, you are performing an in-place operation on your tensor which means their underlying data gets modified without changing the reference two that tensor storage in memory, moreover its metadata remains unchanged, that is W and b are and remain tensors which require gradient (as defined in the very first assignments with requires_grad=True).
On the other, you are performing out-of-place operations which means variables  W and b both get assigned brand new tensors. Indeed, out-of-place assignments create copies. Therefore the W and b are no longer the ones defined prior to the assignment but different ones. Not only their values are different, but the tensors' metadata itself has changed. Finally, the reason why you have None is that tensors defined under this context manager will not have a requires_grad=True set by definition of the context.


Instruction: How to slice 2D Torch tensor individually per row?

Input:
I have a 2D tensor in Pytorch that I would like to slice:
x = torch.rand((3, 5))

In this example, the tensor has 3 rows and I want to slice x, creating a new tensor y that also has 3 rows and num_col cols.
What's challenging for me is that I want to slice different columns per row. All I have is x, num_cols, and idx, which is a tensor holding the start index from where to slice.
Example:
What I have is num_cols=2, idx=[1,2,3] and
x=torch.arange(15).reshape((3,-1)) =
tensor([[ 0,  1,  2,  3,  4],
        [ 5,  6,  7,  8,  9],
        [10, 11, 12, 13, 14]])

What I want is
y=
tensor([[ 1,  2],
        [ 7,  8],
        [13, 14]])

What's the "torch"-way of doing this? I know, I can slice if I get a boolean mask somehow, but I don't know how to construct that with idx and num_cols without normal Python loops.

Output:
You could use fancy indexing together with broadcasting. Another solution might be to use torch.gather which is similar to numpy's take_along_axis. Your idx array would need to be extended with the extra column:
x = torch.arange(15).reshape(3,-1)
idx = torch.tensor([1,2,3])

idx = torch.column_stack([idx, idx+1])
torch.gather(x, 1, idx)

output:
tensor([[ 1,  2],
        [ 7,  8],
        [13, 14]])


Instruction: How is self() used in Pytorch to generate predictions?

Input:
class MNIST_model(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(input_size, num_classes)

    def forward(self, xb):
        xb = xb.reshape(-1, 28 * 28)
        out = self.linear(xb)
        return out

    def training_step(self, batch):
        images, labels = batch
        out = self(images)
        loss = F.cross_entropy(out, labels)
        return loss

I am following the Freecodecamp tutorial.
In the training_step method, the tutorial says
out = self(images) is used to Generate Predictions.
I am not able to understand how is self is being used to get the predictions.

Output:
This is actually nothing specific to PyTorch but rather to how Python works.
Using parenthesis on an object or directly on self inside that class will call a special Python function named __call__. This function is available to your class because you're inheriting from nn.Module which implemented it for you.
Here's a minimal example outside of PyTorch:
class A():
    def __call__(self):
        print('calling the object')

    def foo(self):
        self()

Then
>>> x = A()
>>> x.foo() # prints out "calling the object"


Instruction: Pytorch Python Distributed Multiprocessing: Gather/Concatenate tensor arrays of different lengths/sizes

Input:
If you have tensor arrays of different lengths across several gpu ranks, the default all_gather method does not work as it requires the lengths to be same.
For example, if you have:
if gpu == 0:
    q = torch.tensor([1.5, 2.3], device=torch.device(gpu))
else:
    q = torch.tensor([5.3], device=torch.device(gpu))

If I need to gather these two tensor arrays as follows:
all_q = [torch.tensor([1.5, 2.3], torch.tensor[5.3])
the default torch.all_gather does not work as the lengths, 2, 1 are different.

Output:
As it is not directly possible to gather using built in methods, we need to write custom function with the following steps:

Use dist.all_gather to get sizes of all arrays.
Find the max size.
Pad local array to max size using zeros/constants.
Use dist.all_gather to get all padded arrays.
Unpad the added zeros/constants using sizes found in step 1.

The below function does this:
def all_gather(q, ws, device):
    """
    Gathers tensor arrays of different lengths across multiple gpus

    Parameters
    ----------
        q : tensor array
        ws : world size
        device : current gpu device

    Returns
    -------
        all_q : list of gathered tensor arrays from all the gpus

    """
    local_size = torch.tensor(q.size(), device=device)
    all_sizes = [torch.zeros_like(local_size) for _ in range(ws)]
    dist.all_gather(all_sizes, local_size)
    max_size = max(all_sizes)

    size_diff = max_size.item() - local_size.item()
    if size_diff:
        padding = torch.zeros(size_diff, device=device, dtype=q.dtype)
        q = torch.cat((q, padding))

    all_qs_padded = [torch.zeros_like(q) for _ in range(ws)]
    dist.all_gather(all_qs_padded, q)
    all_qs = []
    for q, size in zip(all_qs_padded, all_sizes):
        all_qs.append(q[:size])
    return all_qs

Once, we are able to do the above, we can then easily use torch.cat to further concatenate into a single array if needed:
torch.cat(all_q)
[torch.tensor([1.5, 2.3, 5.3])

Adapted from: github


Instruction: output with shape [64, 1] doesn't match the broadcast shape [64, 2]

Input:
I got above error when trying to pass weighted class to BCELoss (using pytorch). As you can see below. My model is  Resnet with Sigmoid. I guess the model expect one class value instead of two becouse its Sigmoid.
But which one of the value percentage, I should pass. The percentage of postive value (with 1) or negative (with 0)
class_weights2=[postive/(negtive+postive),negtive/(negtive+postive)]
print(class_weights2)
# [0.3135668226071564, 0.6864331773928436]
class_weights=torch.tensor(class_weights2,dtype=torch.float)
lossFunc= torch.nn.BCELoss(class_weights)

and this the model:
model = torchvision.models.resnet50(pretrained=False)

model.fc = torch.nn.Sequential(
    torch.nn.Linear(
        in_features=2048,
        out_features=1
    ),
    torch.nn.Sigmoid()
)

Output:
The weights passed to BCELoss are not class weights. They rescale the contribution of each element in the batch.
From the docs:

a manual rescaling weight given to the loss of each batch element. If
given, has to be a Tensor of size nbatch.


Instruction: Why tanh function return different in tensorflow and pytorch?

Input:
I find that tensorflow and pytorch tanh result is different, I want to know why did this happen?
I know that the difference is very small, so is this acceptable?
import numpy as np
import tensorflow as tf
import torch

np.random.seed(123)
tf.random.set_seed(123)
torch.manual_seed(123)

batch, sentence_length, embedding_dim = 20, 5, 10
value = np.random.random((batch, sentence_length, embedding_dim)).astype("f")
value = value * 10

tf_x = tf.constant(value, dtype=tf.float32)
tf_out = tf.math.tanh(tf_x)

pt_x = torch.from_numpy(value)
pt_out = torch.tanh(pt_x)

print((tf_out.numpy() == pt_out.numpy()).all()) # return False
print(((tf_out.numpy() - pt_out.numpy()) < 1e-6).all()) # return True

tensorflow == 2.5.0
torch == 1.9.0

Output:
Running your code with the following line at the end:
print(np.allclose(tf_out.numpy(), pt_out.numpy()))  # Returns True

You will receive True. I do not know exactly how tensorflow and pytorch compute the tanh oppeartion, but when working with floating points, you rarely are exactely equal. However, you should be receiving equal results up to a certain tolerance, which is exactly what np.allclose() checks. Read more onallclose here


Instruction: Pytorch: How to make a custom Dataloader for CNN?

Input:
I'm trying to create my own Dataloader from a custom dataset for a CNN. The original Dataloader was created by writing:
train_loader = torch.utils.data.DataLoader(mnist_data, batch_size=64)

If I check the shape of the above, I get
i1, l1 = next(iter(train_loader))
print(i1.shape)   # torch.Size([64, 1, 28, 28])
print(l1.shape)   # torch.Size([64])

When I feed this train_loader into my CNN, it works beautifully. However, I have a custom dataset. I have done the following:
mnist_data = datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor())

trainset = mnist_data
testset = mnist_data

x_train = np.array(trainset.data)
y_train = np.array(trainset.targets)

# modify x_train/y_train

Now, how would I be able to take x_train, y_train and make it into a Dataloader similar to the first one? I have done the following:
train_data = []
for i in range(len(x_train)):
   train_data.append([x_train[i], y_train[i]])

train_loader = torch.utils.data.DataLoader(train_data, batch_size=64)

for i, (images, labels) in enumerate(train_loader):
    images = images.unsqueeze(1)

However, I'm still missing the channel column (which should be 1). How would I fix this?

Output:
I don't have access to your x_train and y_train, but probably this works:
from torch.utils.data import TensorDataset, DataLoader

# use x_train and y_train as numpy array without further modification
x_train = np.array(trainset.data)
y_train = np.array(trainset.targets)

# convert to numpys to tensor
tensor_x = torch.Tensor(x_train)
tensor_y = torch.Tensor(y_train)
# create the dataset
custom_dataset = TensorDataset(tensor_x,tensor_y)
# create your dataloader
my_dataloader = DataLoader(custom_dataset,batch_size=1)

#check if you can get the desired things
i1, l1 = next(iter(my_dataloader))
print(i1.shape)   # torch.Size([1, 1, 28, 28])
print(l1.shape)   # torch.Size([1])


Instruction: Altering pytorch resnet head from sigmoid to Softmax

Input:
I'm new to pytorch. I wrote the below code to do predication using Resnet with Sigmoid for binary classification. I just need to change it to softmax because I might have more than 2 classes.
I understood that pytorch, unlike, Keras, the softmax is in the CrossEntropyLoss. So I'm not sure how could I change the top layer to make the model uses softmax:
model =  torchvision.models.resnet50(pretrained=False)

model.fc = torch.nn.Sequential(
    torch.nn.Linear(
        in_features=2048,
        out_features=1
    ) ,   torch.nn.Sigmoid()
)

model = model.cpu()

and later:
lossFunc=torch.nn.BCELoss(class_weights)

Output:
You can try this:
model.fc[1] = torch.nn.Softmax(10)

where 10 are the number of classes, you can put value based on your needs.