Instruction: How do I make my custom loss function scalar?

Input:
I am coding a dqn from scratch and therefore have written my loss function. While calling backward on my loss function, I get the following error - RuntimeError: grad can be implicitly created only for scalar outputs
Here's my code -
import numpy as np
import gym
import matplotlib.pyplot as plt
import os
import torch
import random
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from collections import deque
import sys
env = gym.make("CliffWalking-v0")

# In[103]:

#Hyperparameters
episodes = 5000
eps = 1.0
learning_rate = 0.1
discount_factor = 0.99
tot_rewards = []
decay_val = 0.001
mem_size = 50000
batch_size = 2
gamma = 0.99

# In[104]:

class NeuralNetwork(nn.Module):
    def __init__(self, state_size, action_size):
        super(NeuralNetwork, self).__init__()
        self.state_size = state_size
        self.action_size = action_size
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(1, 30),
            nn.ReLU(),
            nn.Linear(30, 30),
            nn.ReLU(),
            nn.Linear(30, action_size)
        )
    def forward(self, x):
        x = self.linear_relu_stack(x)
        return x

# In[105]:

model = NeuralNetwork(env.observation_space.n, env.action_space.n)
opt = torch.optim.Adam(params=model.parameters(), lr=learning_rate)
loss = nn.MSELoss()
replay_buffer = deque(maxlen=mem_size)

# In[106]:

state = torch.tensor(env.reset(), dtype=torch.float32)
state = state.unsqueeze(dim=0)
print(state.shape)
out = model(state)

# In[111]:

def compute_td_loss(batch_size):
    state, next_state, reward, done, action = zip(*random.sample(replay_buffer, batch_size))
    state = torch.from_numpy(np.array(state).reshape(-1, 1)).unsqueeze(dim = 0).type(torch.float32)
    next_state = torch.from_numpy(np.array(next_state).reshape(-1, 1)).unsqueeze(dim = 0).type(torch.float32)
    reward = torch.from_numpy(np.array(reward))
    done = torch.from_numpy(np.array(done))
    action = torch.from_numpy(np.array(action)).type(torch.int64)
    q_values = model(state)
    next_q_values = model(next_state)
    q_vals = q_values.squeeze().gather(dim=-1, index=action.reshape(-1,1)).reshape(1, -1)
    max_next_q_values = torch.max(next_q_values,2)[0].detach()
    print("q_vals = ", q_vals)
    print("max_next_q_values = ", max_next_q_values)
    loss = 0.5*(reward + gamma*max_next_q_values - q_vals)**2
    print("reward = ", reward)
    print("loss = ", loss)
    opt.zero_grad()
    loss.backward()
    opt.step()
    return loss

# In[112]:

for i in range(episodes):
    state = env.reset()
    done = False
    steps = 0
    eps_rew = 0
    while not done and steps<50:
        if np.random.uniform(0,1)<eps:
            action = env.action_space.sample()
        else:
            state = torch.tensor(state, dtype=torch.float32)
            state = state.unsqueeze(dim=0)
            action = np.argmax(model(state).detach().numpy())
        next_state, reward, done, info = env.step(action)
        replay_buffer.append((state, next_state, reward, done, action))
        if len(replay_buffer)>batch_size:
            loss = compute_td_loss(batch_size)
            sys.exit()
        eps = eps/(1 + 0.001)
        eps_rew += reward
        if done:
            break
        state = next_state
    tot_rewards.append(eps_rew)

Here's the error that I get -
RuntimeError                              Traceback (most recent call last)
<ipython-input-112-015fd74c95d9> in <module>
     14         replay_buffer.append((state, next_state, reward, done, action))
     15         if len(replay_buffer)>batch_size:
---> 16             loss = compute_td_loss(batch_size)
     17             sys.exit()
     18         eps = eps/(1 + 0.001)

<ipython-input-111-3e1e02c32b4f> in compute_td_loss(batch_size)
     16     print("loss = ", loss)
     17     opt.zero_grad()
---> 18     loss.backward()
     19     opt.step()
     20     return loss

c:\users\thoma\anaconda3\envs\custom_atari_env\lib\site-packages\torch\_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    253                 create_graph=create_graph,
    254                 inputs=inputs)
--> 255         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    256
    257     def register_hook(self, hook):

c:\users\thoma\anaconda3\envs\custom_atari_env\lib\site-packages\torch\autograd\__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    141
    142     grad_tensors_ = _tensor_or_tensors_to_tuple(grad_tensors, len(tensors))
--> 143     grad_tensors_ = _make_grads(tensors, grad_tensors_)
    144     if retain_graph is None:
    145         retain_graph = create_graph

c:\users\thoma\anaconda3\envs\custom_atari_env\lib\site-packages\torch\autograd\__init__.py in _make_grads(outputs, grads)
     48             if out.requires_grad:
     49                 if out.numel() != 1:
---> 50                     raise RuntimeError("grad can be implicitly created only for scalar outputs")
     51                 new_grads.append(torch.ones_like(out, memory_format=torch.preserve_format))
     52             else:

RuntimeError: grad can be implicitly created only for scalar outputs

Output: Given that your batch_size = 2 and looking at your code your loss will likely be of size batch_size x 1. Given that what you are likely trying to do is to compute the gradient of expected Q loss you can use monte carlo estimator where instead of computing an expectation we use a mean over a finite sample (here - your batch). Consequently what you are missing is taking a mean of your loss before calling backwards.


Instruction: RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed)

Input:
I am trying to get to grips with Pytorch and I wanted to try to reproduce this code:
https://github.com/andy-psai/MountainCar_ActorCritic/blob/master/RL%20Blog%20FINAL%20MEDIUM%20code%2002_12_19.ipynb
in Pytorch.
I am having a problem in that this error is being returned:
RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.
A similar question said to use zero_grad() again after the optimizer step, but this hasn't resolved the issue.
I've included the entire code below so hopefully it should be reproduceable.
Any advice would be much appreciated.
import gym
import os
import os.path as osp
import time
import math
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal

env = gym.envs.make("MountainCarContinuous-v0")

# Value function

class Value(nn.Module):
    def __init__(self, dim_states):
        super(Value, self).__init__()
        self.net = nn.Sequential(
        nn.Linear(dim_states, 400),
        nn.ReLU(),
        nn.Linear(400,400),
        nn.ReLU(),
        nn.Linear(400, 1)
    )
        self.optimizer = optim.Adam(self.parameters(), lr = 1e-3)
        self.criterion = nn.MSELoss()

    def forward(self, state):
        return self.net(torch.from_numpy(state).float())

    def compute_return(self, output, target):
        self.optimizer.zero_grad()
        loss = self.criterion(output, target)
        loss.backward()
        self.optimizer.step()
        self.optimizer.zero_grad()

# Policy network

class Policy(nn.Module):
    def __init__(self, dim_states, env):
        super(Policy, self).__init__()

        self.hidden1 = nn.Linear(dim_states, 40)
        self.hidden2 = nn.Linear(40, 40)
        self.mu = nn.Linear(40, 1)
        self.sigma = nn.Linear(40,1)
        self.env = env

        self.optimizer = optim.Adam(self.parameters(), lr = 2e-5)

    def forward(self, state):
        state = torch.from_numpy(state).float()
        x = F.relu(self.hidden1(state))
        x = F.relu(self.hidden2(x))
        mu = self.mu(x)
        sigma = F.softmax(self.sigma(x), dim=-1)
        action_dist = Normal(mu, sigma)
        action_var = action_dist.rsample()
        action_var = torch.clip(action_var,
                        self.env.action_space.low[0],
                        self.env.action_space.high[0])

        return action_var, action_dist

    def compute_return(self, action, dist, td_error):
        self.optimizer.zero_grad()
        loss_actor = -dist.log_prob(action)*td_error
        loss_actor.backward()
        self.optimizer.step()
        self.optimizer.zero_grad()

# Normalise the state space
import sklearn
import sklearn.preprocessing

state_space_samples = np.array(
    [env.observation_space.sample() for x in range(10000)])
scaler = sklearn.preprocessing.StandardScaler()
scaler.fit(state_space_samples)

# Normaliser
def scale_state(state):
    scaled = scaler.transform([state])
    return scaled
##################################

# Parameters

lr_actor = 0.00002
lr_critic = 0.001

actor = Policy(2, env)
critic = Value(2)

# Training loop params
gamma = 0.99
num_episodes = 300

episode_history = []

for episode in range(num_episodes):

    # Receive initial state from E

    state = env.reset()
    reward_total = 0
    steps = 0
    done = False

    while not done:

        action, dist = actor(state)

        # print(np.squeeze(action))
        next_state, reward, done, _ = env.step(
                                        np.array([action.item()]))

        if episode % 50 == 0:
            env.render()

        steps += 1
        reward_total += reward

        # TD Target
        target = reward + gamma * np.squeeze(critic(next_state), axis=0)

        td_error = target - np.squeeze(critic(state), axis=0)

        # Update actor
        actor.compute_return(action, dist, td_error)

        # Update critic
        critic.compute_return(np.squeeze(critic(state), axis=0), target)

    episode_history.append(reward_total)

    print(f"Episode: {episode}, N Steps: {steps}, Cumulative reward {reward_total}")

    if np.mean(episode_history[-100:]) > 90 and len(episode_history) > 101:
        print("Solved")
        print(f"Mean cumulative reward over 100 episodes {np.mean(episode_history[-100:])}")

Output:
Problem lies in this snippet. When you create target variable, there is a forward pass through critic which generates a computation graph and critic(next_state) is the leaf node of that graph making target a part of the graph (you can check this by printing target which will show you grad_fn=<AddBackward0>). Finally, when you call critic.compute_return(critic_out, target), a new computation graph is generated and passing target(which is a part of the previous computation graph) causes a Runtime error.
Solution is to call detach() on critic(next_state), this will free target variable and it will no longer be a part of the computation graph(again check by printing target).
target = reward + gamma * np.squeeze(critic(next_state).detach(), axis=0)
td_error = target - np.squeeze(critic(state), axis=0)

# Update actor
actor.compute_return(action, dist, td_error)

# Update critic
critic_out = np.squeeze(critic(state), axis=0)
print(critic_out)
critic.compute_return(critic_out, target)


Instruction: Most efficient way to use Column major when reshape a 1D array in PyTorch

Input:
import torch

    p = torch.arange(0, 12, requires_grad=False, dtype=torch.int32)
    pr = torch.reshape(p, (4, 3))

what I want is
pr = [0 4 8
      1 5 9
      2 6 10
      3 7 11]

but it actually becomes
pr = [0 1 2
      3 4 5
      6 7 8
      9 10 11]

I search online it said permute can do it, but it will make a copy of your array, what is the most efficient way to reshape it in PyTorch?

Output:
You are close but to get what you want you have to rearrange it a little bit.
import torch

p = torch.arange(0, 12, requires_grad=False, dtype=torch.int32)
pr = torch.reshape(p, (3, 4)).t()

pr
Out[49]:
tensor([[ 0,  4,  8],
        [ 1,  5,  9],
        [ 2,  6, 10],
        [ 3,  7, 11]], dtype=torch.int32)


Instruction: How to convert a generator to a Pytorch Dataloader?

Input: I have a generator that creates synthetic data. How can I convert this into a PyTorch dataloader?

Output:
You can wrap your generator with a data.IterableDataset:
class IterDataset(data.IterableDataset):
    def __init__(self, generator):
        self.generator = generator

    def __iter__(self):
        return self.generator()

Naturally, you can then wrap this dataset with a data.DataLoader.
Here is a minimal example showing its use:
>>> gen = lambda: [(yield x) for x in range(10)]

>>> dataset = IterDataset(gen)
>>> for i in data.DataLoader(dataset, batch_size=2):
...    print(i)
tensor([0, 1])
tensor([2, 3])
tensor([4, 5])
tensor([6, 7])
tensor([8, 9])


Instruction: How to set specific values for the weight and bias in a neural net?

Input:
I want to initialize the values of the weight and bias of the linear layers in my PyTorch neural network. Below is some code for my neural net:
class NeuralNet(nn.Module):

    def __init__(self, weights, bias):

        super(NeuralNet, self).__init__()

        self.weights = weights
        self.bias = bias

        self.nn = nn.Sequential(
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 1),
            nn.ReLU(),
        )

    def forward(self, a, b, c):

        a = torch.flatten(a) # shape: (n,)
        b = torch.flatten(b) # shape: (n, 1)
        c = torch.flatten(c) # shape: (n,)

        y = torch.stack((a, b, c), 1)

        y1 = self.nn(y)

        return y1

weights = torch.rand(5)
bias = torch.rand(5)

net = NeuralNet(weights, bias)

Based on my understanding, aach layer in the neural net is currently related to 5 parameters (weights, bias, a, b, and c). Let's say I want to assign a list of values of the weights and biases that I know are pretty close to the actual values to the corresponding layer in the neural net. How do I go about doing that?
To clarify:
The PyTorch documentation says that nn.Linear contains two variables weight (~Linear.weight) and bias (~Linear.bias). I want to be able to assign values to each of these two values for every layer inside my neural net. Is there a way to point to each linear layer inside of nn.Sequential and set a value for the weights and bias?

Output:
First, when assigning a weight value to a linear layer, the matrix size must be the same. The weight matrix size of fc1, fc2, and fc3 is 3*3, and the weight matrix size of fc4 is 1*3.
Second, to change the weight value, use no grad to prevent the weight from changing during training.
Third, when accessing sequential layers, you need to use square brackets. In the code below, the 0th element of self.l is nn.linear, the 1st element is nn.relu, the 2nd element is nn.linear, and so on.
Finally, you need to change the weight value using torch.nn.Parameter.
Code:
import torch
import torch.nn as nn
import torch.nn.functional as F

class NeuralNet(nn.Module):
    def __init__(self):
        super(NeuralNet, self).__init__()
        self.l = nn.Sequential(
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 3),
            nn.ReLU(),
            nn.Linear(3, 1)
        )

    def forward(self, a, b, c):
        a = torch.flatten(a)
        b = torch.flatten(b)
        c = torch.flatten(c)
        y = torch.stack((a, b, c), 1)

        y = self.l(y)

        return y

net = NeuralNet()
with torch.no_grad():
    print(net.l[0].weight)
    net.l[0].weight = nn.Parameter(torch.randn(3,3))
    net.l[0].bias = nn.Parameter(torch.randn(3))
    print(net.l[0].weight)

Result:
Parameter containing:
tensor([[ 0.0684, -0.2947,  0.0059],
        [-0.0430, -0.0716, -0.1406],
        [-0.4114, -0.3912, -0.5121]], requires_grad=True)
Parameter containing:
tensor([[-0.5050, -0.1787, -0.5368],
        [ 0.6825, -0.5514,  2.1333],
        [-1.5563,  2.4367, -0.6201]], requires_grad=True)


Instruction: AMD ROCm with Pytorch on Navi10 (RX 5700 / RX 5700 XT)

Input:
I am one of those miserable creatures who own a AMD GPU(RX 5700, Navi10). I want to use up-to-date PyTorch libraries to do some Deep Learning on my local machine and stop using cloud instances.
I saw all over the internet that AMD is promising Navi10 support in the next 2-4 months(posts that were written 1-2 years back) however, I do not think they released an "official" support.
I installed ROCm on local machine and it actually detects my GPU and everything seems nice, here is rocminfo output.
rocminfo output
I installed the necessary PyTorch ROCm version but when I try to run a code, I get the following error.
Error mesage
"hipErrorNoBinaryForGpu: Unable to find code object for all current devices!"
I suppose this is because ROCm still does not have a support for gfx1010 or I am lost at this point.
I would be happy if someone can provide a way to make ROCm work(preferable without compiling whole package for gfx1010 again) OR provide way to use an AMD GPU just like a CUDA user.

Output:
Add 'HSA_OVERRIDE_GFX_VERSION=10.3.0' before 'python'
for example, in teminal, input:
HSA_OVERRIDE_GFX_VERSION=10.3.0 python launch.py
I used 5700xt to run stable-diffusion for months, it works.


Instruction: How can I train a model with untrained feature extraction layers? (PyTorch)

Input:
I was wondering how can I keep my batch norm layers active using an untrained feature extraction network.
Would this be considered feature extraction with an "untrained" network?:
class DenseNetConv(torch.nn.Module):
    def __init__(self):
        super(DenseNetConv,self).__init__()
        original_model = models.densenet161(pretrained=False)
        self.features = torch.nn.Sequential(*list(original_model.children())[:-1])
        self.avgpool = nn.AdaptiveAvgPool2d(1)
        for param in self.parameters():
            param.requires_grad = False

    def forward(self, x):
        x = self.features(x)
        x = F.relu(x, inplace=True)
        x = F.avg_pool2d(x, kernel_size=7).view(x.size(0), -1)
        return x

The above should return a tensor of [batch size, 2208], however, I want to make sure that by stating pretrained=False that I am basically extracting features from an untrained network.
I then use the following to define the classifier layers:
class MyDenseNetDens(torch.nn.Module):
    def __init__(self, nb_out=2):
        super().__init__()
        self.dens1 = torch.nn.Linear(in_features=2208, out_features=512)
        self.dens2 = torch.nn.Linear(in_features=512, out_features=128)
        self.dens3 = torch.nn.Linear(in_features=128, out_features=nb_out)

    def forward(self, x):
        x = self.dens1(x)
        x = torch.nn.functional.selu(x)
        x = F.dropout(x, p=0.25, training=self.training)
        x = self.dens2(x)
        x = torch.nn.functional.selu(x)
        x = F.dropout(x, p=0.25, training=self.training)
        x = self.dens3(x)
        return x

and finally join them together here:
class MyDenseNet(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.mrnc = MyDenseNetConv()
        self.mrnd = MyDenseNetDens()
    def forward(self, x):
        x = self.mrnc(x)
        x = self.mrnd(x)
        return x

densenet = MyDenseNet()
densenet.to(device)
densenet.train()

If I allow this to train, for example by applying densenet.train() will this be sufficient in allowing batch normalisation statistics to be generated for each mini-batch as well as allow for the running means and standard deviations to be learnt and applied during inference, while keeping the convolutional layers untrained?

Output:
Unfortunately, it will update the running stat anyway since the default batchnorm in your self.mrnc still initialized with the running mean and the running variance. All you need to do is shut them down:
class MyDenseNetConv(torch.nn.Module): # I renamed it to be correct to your code
    def __init__(self):
        super(MyDenseNetConv,self).__init__()
        original_model = models.densenet161(pretrained=False)
        for n, v in original_model.named_modules(): # this part is to remove the tracking stat option
            if 'norm' in n:
                v.track_running_stats=False
                v.running_mean = None
                v.running_var = None
                v.num_batches_tracked = None
        self.features = torch.nn.Sequential(*list(original_model.children())[:-1])
        self.avgpool = nn.AdaptiveAvgPool2d(1)
        for param in self.parameters():
            param.requires_grad = False

    def forward(self, x):
        x = self.features(x)
        x = F.relu(x, inplace=True)
        x = F.avg_pool2d(x, kernel_size=7).view(x.size(0), -1)
        return x


Instruction: How to get the gradients from loss to target layer output for Grad-CAM computation with pytorch

Input:
I want to implement the code to get Grad-CAM map with pytorch(1.10.0). Most of implementation specify the target class to extract the gradients (This is a natural approach). But instead of this, I want to use gradients from loss to target layer output.
The following code is an example of a keras implementation of what I want to do. How can I implement it with pytorch?
loss = K.mean(K.square(y_pred - y_true), axis=-1)
output_conv = model.get_layer(layer_name).output
gradients = K.gradients(loss, output_conv)[0]

output, grads = K.function([model.input], [output_conv, gradients])([input_image])

Output:
You can register the hook for both the forward and backward of a model while inferencing. I wrote an easy example with ResNet50:
import torch
from torchvision import models

activation = {}
def get_forward(name):
    def hook(model, input, output):
        activation[name] = output.detach()
    return hook

def get_backward(name):
    def hook(model, input, output):
        activation[name] = output[0].detach()
    return hook

model = models.resnet50(pretrained=False)
model.layer4[2].register_forward_hook(get_forward('forward'))
model.layer4[2].register_backward_hook(get_backward('backward'))

x = torch.randn(10, 3, 224, 224)
out = model(x)
loss = out.mean()
loss.backward()

output, grads = activation['forward'], activation['backward']
print(output.shape)
print(grads.shape)

>>> torch.Size([10, 2048, 7, 7])
>>> torch.Size([10, 2048, 7, 7])

The function should work well with CNNs, be careful with the Linear layer where the hook doesn't work as expected because of this bug which is simplified by this answer


Instruction: Sort a multi-dimensional tensor using another tensor

Input:
I'm trying to sort C (see image) using R to get Sorted_C.

c = torch.tensor([[[0, 1, 0, 0, 0], [1, 0, 1, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 1]], [[0, 0, 1, 1, 0], [1, 1, 0, 0, 0], [0, 0, 1, 0, 0], [0, 1, 0, 1, 0]]])
r = torch.tensor([[[0, 0, 0, 7.2, 0], [0, 25.4, 0, 0, 0], [0, 0, 43.6, 0, 0], [61.8, 0, 0, 0, 0], [0, 0, 0, 0, 80]], [[0, 0, 0, 0, 98.2], [116.4, 0, 0, 0, 0], [0, 134.6, 0, 0, 0], [0, 0, 152.8, 0, 0], [0, 0, 0, 169.2, 0]]])

# this is what I need
sorted_c = torch.tensor([[[0, 1, 0, 0, 0], [0, 0, 1, 1, 0], [1, 0, 0, 0, 0], [0, 0, 0, 0, 1]], [[0, 0, 0, 1, 1], [0, 1, 1, 0, 0], [0, 0, 0, 1, 0], [0, 0, 1, 0, 1]]])

How do I do this efficiently?
correction: expected ordering for P6 to P10 should be: P10 -> P6 -> P7 -> P8 -> P9

Output:
Minor note: it seems your example r doesn't quite match the values in your drawing. Also, it seems the result for p6 through p10 should be p10 < p6 < p7 < p8 < p9.

When you hear "advanced indexing", you should think torch.gather. That is: the resulting tensor comes from the indexing of c column-wise with some kind of tensor we will extract from r.
First we can sum and sort r to get the indices of the columns:
>>> idx = r.sum(1).argsort(1)
tensor([[3, 1, 2, 0, 4],
        [4, 0, 1, 2, 3]])

Then we can apply torch.Tensor.gather indexing c column-wise using the column indices contained in idx i.e. dim=2 is the one varying based on values in idx. Explicitly the resulting tensor out is constructed such that:
out[i][j][k] = c[i][j][idx[i][j][k]]

Keep in mind both the index tensor (idx) and the value tensor (c) must have same dimension sizes except for the one that we're indexing on, here dim=2. Therefore, we need to expand idx such that it has the shape of c. We can do so with None-indexing and using expand or expand_as:
>>> idx[:,None].expand_as(c)
tensor([[[3, 1, 2, 0, 4],
         [3, 1, 2, 0, 4],
         [3, 1, 2, 0, 4],
         [3, 1, 2, 0, 4]],

        [[4, 0, 1, 2, 3],
         [4, 0, 1, 2, 3],
         [4, 0, 1, 2, 3],
         [4, 0, 1, 2, 3]]])

Notice the duplicated values row-wise (fiy: they're not copies, expand is makes a view not a copy!)
Finally, we can gather the values in c to get the desired result:
>>> c.gather(2, idx[:,None].expand_as(c))
tensor([[[0, 1, 0, 0, 0],
         [0, 0, 1, 1, 0],
         [1, 0, 0, 0, 0],
         [0, 0, 0, 0, 1]],

        [[0, 0, 0, 1, 1],
         [0, 1, 1, 0, 0],
         [0, 0, 0, 1, 0],
         [0, 0, 1, 0, 1]]])


Instruction: Huggingface Trainer load_best_model f1 score vs. loss and overfitting

Input:
I have trained a roberta-large and specified load_best_model_at_end=True and metric_for_best_model=f1. During training, I can see overfitting after the 6th epoch, which is the sweetspot. In Epoch 8, which is the next one to evaluate due to gradient accumulation, we can see that train loss decreases and eval_loss increases. Thus, overfitting starts. The transformers trainer in the end loads the model from epoch 8, checkpoint -14928, as the f1 score is a bit highea. I was wondering, in theory, wouldn't be the model from epoch 6 be better suited, as it did not overfit? Or does one really go for the f1 metric here even though the model did overfit? (the eval loss decreased in epochs <6 constantly).
The test_loss from the second checkpoint, which is then loaded as the "best", is 0.128. Is it possible to lower that using the first checkpoint which should be the better model anyway?
checkpoint-11196:
{'loss': 0.0638, 'learning_rate': 8.666799323450404e-06, 'epoch': 6.0}

{'eval_loss': 0.09599845856428146, 'eval_accuracy': 0.9749235986101227, 'eval_precision': 0.9648319293367138, 'eval_recall': 0.9858766505097777, 'eval_f1': 0.9752407721241682, 'eval_runtime': 282.2294, 'eval_samples_per_second': 84.637, 'eval_steps_per_second': 2.647, 'epoch': 6.0}

VS.

checkpoint-14928:
{'loss': 0.0312, 'learning_rate': 7.4291115311909265e-06, 'epoch': 8.0}

{'eval_loss': 0.12377820163965225, 'eval_accuracy': 0.976305103194206, 'eval_precision': 0.9719324391455539, 'eval_recall': 0.9810295838208257, 'eval_f1': 0.9764598236566295, 'eval_runtime': 276.7619, 'eval_samples_per_second': 86.309, 'eval_steps_per_second': 2.699, 'epoch': 8.0}

Output: You could just comment the metric_for_best_model='f1' part out and see for yourself, loss is the default setting. Or, utilize from_pretrained('path/to/checkpoint') to compare two checkpoints back to back. F-score is threshold sensitive, so it's entirely possible for a lower loss checkpoint to be better in the end (assuming you do optimize the threshold).


Instruction: I want to deploy a pytorch segmentation model in a C++ application .. C++ equivalent preprocessing

Input:
I want to deploy a pytorch segmentation model in a C++ application. I knew that I have to convert the model to a Torch Script and use libtorch.
However, what is C++ equivalent to the following pre-preprocessing (It's Ok to convert opencv, but I don't know how to convert the others)?
    import torch.nn.functional as F

    train_tfms = transforms.Compose([transforms.ToTensor(), transforms.Normalize(channel_means, channel_stds)])
    input_width, input_height = input_size[0], input_size[1]

    img_1 = cv.resize(img, (input_width, input_height), cv.INTER_AREA)
    X = train_tfms(Image.fromarray(img_1))
    X = Variable(X.unsqueeze(0)).cuda()  # [N, 1, H, W]

    mask = model(X)

    mask = F.sigmoid(mask[0, 0]).data.cpu().numpy()
    mask = cv.resize(mask, (img_width, img_height), cv.INTER_AREA)

Output:
To create the transformed dataset, you will need to call MapDataset<DatasetType, TransformType> map(DatasetType dataset,TransformType transform) (see doc).
You will likely have to implement your 2 transforms yourself, just look at how they implemented theirs and imitate that.
The libtorch tutorial will guide you through datasets and dataloaders
You can call the sigmoid function with torch::nn::functionql::sigmoid I believe


Instruction: What does out_channels in Conv2d represent?

Input:
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()

I am taking a look at PyTorch Blitz and in the conv1 layer we can see the input_channels=3 because it's the first image so it just has its 3 RGB channels and out_channels=6.
Does that mean the number of filters I have are 6? In which case it'd mean the total number of feature maps I would get are 6*3==18? But if that is the case why in conv2 am I plugging in input_channels=6, shouldn't I be plugging in  18 because that was the output from the previous Convolutional layer?

Output:
No, you get the number of out_channels (that IS the number of feature maps). So, if you start of with 3 representations of your data - i.e., your input channels, you can chose how many representations you want after the convolutional operation, in this case you specified 6. Naturally, in the next layer, you'll have 6 as input as a result.
It sounds like you don't fully grasp how convolutional neural networks work internally, look at some explanations like here.
If you apply your case to the picture from that website, you'd have 6 different filters, leading to 6 output channels. The number you pick, is up to you!


Instruction: PyTorch: Fully Connected Layer has no Parameters

Input:
I am very new at PyTorch so please excuse my ignorance.  I am trying to create my own CNN using PyTorch.  The problem is that my Fully Connected Layer (i.e. the LazyLinear function) is showing no learnable parameters and the network is obviously not learning anything.
import torch
from torch import nn
import pytorch_lightning as pl
from pytorch_lightning.core.decorators import auto_move_data

class ThreeConvLayer(pl.LightningModule):

def __init__(self, num_classes, num_images):
  super(ThreeConvLayer, self).__init__()
  self.convolutionlayer1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, padding=1)
  self.BatchNormalization1 = nn.BatchNorm2d(16)
  self.ReLU1 = nn.ReLU()

  self.maxpool1 = nn.MaxPool2d(kernel_size = 2, stride = 2)

  self.convolutionlayer2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding=1)
  self.BatchNormalization2 = nn.BatchNorm2d(32)
  self.ReLU2 = nn.ReLU()

  self.maxpool2 = nn.MaxPool2d(kernel_size = 2, stride=2)

  self.convolutionlayer3 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1)
  self.BatchNormalization3 = nn.BatchNorm2d(64)
  self.ReLU3 = nn.ReLU()

  self.fc1 = nn.LazyLinear(num_classes)
  self.softmax = nn.Softmax(dim=1)
  self.loss = nn.CrossEntropyLoss()

def forward(self, x):
  #print('shape x:  ', x.shape)
  out = self.convolutionlayer1(x)
  out = self.BatchNormalization1(out)
  out = self.ReLU1(out)
  out = self.maxpool1(out)
  # print('shape out1:  ', out.shape)

  out = self.convolutionlayer2(out)
  out = self.BatchNormalization2(out)
  out = self.ReLU2(out)
  out = self.maxpool2(out)
  #print('shape out2:  ', out.shape)

  out = self.convolutionlayer3(out)
  out = self.BatchNormalization3(out)
  out = self.ReLU3(out)
  #print('shape out3:  ', out.shape)

  out = out.reshape(out.size(0), -1)
  #print('out after reshpae:  ', out.shape)

  out = self.fc1(out)
  out = self.softmax(out)
  #print('final out:  ', out.shape)

  return out

def training_step(self, batch, batch_no):
  # implement single training step
  x, y = batch
  y = y.long()
  logits = self(x)
  loss = self.loss(logits, y)
  self.log('val_loss', loss)
  return loss

def configure_optimizers(self):
  # choose your optimizer
  return torch.optim.RMSprop(self.parameters(), lr=0.05)  #lr=0.005

Here is the PyTorch printout on learnable parameters:
   | Name                | Type             | Params
   ----------------------------------------------------------
   0  | convolutionlayer1   | Conv2d           |  160
   1  | BatchNormalization1 | BatchNorm2d      | 32
   2  | ReLU1               | ReLU             | 0
   3  | maxpool1            | MaxPool2d        | 0
   4  | convolutionlayer2   | Conv2d           | 4.6 K
   5  | BatchNormalization2 | BatchNorm2d      | 64
   6  | ReLU2               | ReLU             | 0
   7  | maxpool2            | MaxPool2d        | 0
   8  | convolutionlayer3   | Conv2d           | 18.5 K
   9  | BatchNormalization3 | BatchNorm2d      | 128
   10 | ReLU3               | ReLU             | 0
   11 | fc1                 | LazyLinear       | 0
   12 | softmax             | Softmax          | 0
   13 | loss                | CrossEntropyLoss | 0
   ----------------------------------------------------------

Here you can see that the FullyConnected Layer has no parameters which is clearly wrong.  What did I do wrong here?

Output:
This is one limitation of lazy modules such as LazyLinear. If you read through the documentation of nn.modules.lazy.LazyModuleMixin, you will see:

Modules that lazily initialize parameters, or “lazy modules”, derive the shapes of their parameters from the first input(s) to their forward method. Until that first forward they contain torch.nn.UninitializedParameters that should not be accessed or used, and afterward they contain regular torch.nn.Parameters.

In other words, you need to first perform a dry run with random data before you can proceed with your calls. This is required because lazy modules infer the missing arguments (such as in_features for nn.Linear) after the first inference call, i.e. based on the shape of the first input it receives.

getting the number of parameters registered.

properly register them inside an optimizer.


Instruction: How do the transposed convolutional layer forward and backward passes work?

Input:
Before I present my problem, I want to apologize for those of you that feel this is more of a math post than a programming post. Neural networks are both mathematic and programming heavy, and I felt my problem was in the programming side. I have created a CNN from scratch in c++ (that works). For this reason, I feel as though the functions I use to create a convolution and a full convolution are correct. Programmatically below, I am going to show how I do the basic forward and backward of a CNN with a convolutional layer being the forward:
Matrix<float> cnn_forward(Matrix<float> weight, Matrix<float> prev){
    Matrix<float> output = prev.convolute(weight);
    return output;
}

And the backward pass (I am not using a bias or activation function in this case):
cnn_back cnn_backward(Matrix<float> a_prev, Matrix<float> dz, Matrix<float> kernel){
    Matrix<float> rotated = kernel.rotate_180();
    Matrix<float> dx = dz.convolute_full(rotated);
    Matrix<float> dw = a_prev.convolute(dz);
    cnn_back output;
    output.dw = std::move(dw);
    output.dx = std::move(dx);
    return output;
}

Everything that I have seen online says that the transposed convolutional layer is just the reverse of the convolutional layer. So, I have tried implementing the following as the forward and backward passes of a transposed convolutional layer.
//forward
Matrix<float> fcn_forward(Matrix<float> weight, Matrix<float> prev){
    Matrix<float> output = prev.convolute_full(weight.rotate_180());
    return output;
}

//backward
fcn_back fcn_backward(Matrix<float> a_prev, Matrix<float> dz, Matrix<float> kernel){
    Matrix<float> dx = dz.convolute(kernel);
    Matrix<float> dw = dz.convolute(a_prev);
    fcn_back output;
    output.dw = std::move(dw);
    output.dx = std::move(dx);
    return output;
}
//again, not using a bias or activation function

My goal is to basically implement torch.nn.ConvTranspose2d from pytorch with 2 dimensional matrices. I was hoping to parallel it to the basic convolution formula that I have above.
~EDIT~
This would be the translation into python using numpy arrays which is pretty much an exact replica of my c++ code.
def convolute(X, W, strides=(1,1)):
    new_row = (int)((X.shape[0] - W.shape[0])/strides[0] +1)
    new_col = (int)((X.shape[1] - W.shape[1])/strides[1] +1)
    out = np.zeros((new_row, new_col), dtype=float)
    x_last = 0
    y_last = 0
    for x in range(0, X.shape[0]-(W.shape[0] - 1), strides[0]):
        for y in range(0, X.shape[1]-(W.shape[1] - 1), strides[1]):
            amt = 0.0
            for i in range(0, W.shape[0]):
                for j in range(0, W.shape[1]):
                    amt += W[i][j] * X[x+i][y+j]
            out[x_last][y_last] = amt
            y_last += 1
        x_last += 1
        y_last = 0
    return out

def convolute_full(X, W, strides=(1, 1)):
    row_num = (X.shape[0] - 1) * strides[0] + W.shape[0]
    col_num = (X.shape[1] - 1) * strides[1] + W.shape[1]
    output = np.zeros([row_num, col_num])
    for i in range(0, X.shape[0]):
        i_prime = i * strides[0]
        for j in range(0, X.shape[1]):
            j_prime = j * strides[1]
            for k_row in range(W.shape[0]):
                for k_col in range(W.shape[1]):
                    output[i_prime+k_row, j_prime+k_col] += W[k_row, k_col] * X[i, j]
    return output

def get_errors(predicted, label):
    return label - predicted

def fcn_forward(weight, prev):
    rotated = np.rot90(np.rot90(weight))
    output = convolute_full(prev, rotated)
    return output

def fcn_backward(a_prev, dz, kernel):
    dx = convolute(dz, kernel)
    dw = convolute(dz, a_prev)
    dx = np.clip(dx, 10, -10)
    return dx, dw

def forward(weights, X_init):
    values = []
    values.append(X_init)
    predicted = fcn_forward(weights[0], X_init)
    values.append(predicted)
    predicted = fcn_forward(weights[1], predicted)
    values.append(predicted)
    return values

def backward(weights, values, label, learningRate=0.001):
    dz = get_errors(values[-1], label)
    dx, dw = fcn_backward(values[-2], dz, weights[-1])
    weights[-1] = weights[-1] - learningRate*dw
    dz = dx
    dx, dw = fcn_backward(values[-3], dz, weights[-2])
    weights[-2] = weights[-2] - learningRate*dw
    return weights

def train_example():
    epoch = int(input("enter epoch: "))
    #creating a random input
    inp = np.random.randn(10,10)
    #creating the weight matricies
    weights = [np.random.randn(3,3), np.random.randn(3,3)]
    #creating the wanted output
    label = np.random.randn(14,14)
    for i in range(0, epoch):
        values = forward(weights, inp)
        if(i == 0 or i == 1):
            errors = get_errors(values[-1], label)
            print("errors:")
            print(errors)
            print("error sum: ", np.sum(errors))
        weights = backward(weights, values, label)
    print("current prediction:")
    print(values[-1])
    print("label: ")
    print(label)
    errors = get_errors(values[-1], label)
    print("errors:")
    print(errors)
    print("error sum at end of training: ", np.sum(errors))

Basically, this does not work. The weights are not corrected in the correct way. The errors only continue to get larger (the opposite of the wanted result). What is the correct way to forward and backward propagate a transposed convolutional layer?
EDIT
This is the answer for anyone wondering how it relates to my code above, due to @Bob's answer:
def convolute(X, W, strides=(1,1)):
    new_row = (int)((X.shape[0] - W.shape[0])/strides[0] +1)
    new_col = (int)((X.shape[1] - W.shape[1])/strides[1] +1)
    out = np.zeros((new_row, new_col), dtype=float)
    x_last = 0
    y_last = 0
    for x in range(0, X.shape[0]-(W.shape[0] - 1), strides[0]):
        for y in range(0, X.shape[1]-(W.shape[1] - 1), strides[1]):
            amt = 0.0
            for i in range(0, W.shape[0]):
                for j in range(0, W.shape[1]):
                    amt += W[i][j] * X[x+i][y+j]
            out[x_last][y_last] = amt
            y_last += 1
        x_last += 1
        y_last = 0
    return out

#this is the same result as scipy.signal.convolute2d
def convolute_full(X, W, strides=(1, 1)):
    row_num = (X.shape[0] - 1) * strides[0] + W.shape[0]
    col_num = (X.shape[1] - 1) * strides[1] + W.shape[1]
    output = np.zeros([row_num, col_num])
    for i in range(0, X.shape[0]):
        i_prime = i * strides[0]
        for j in range(0, X.shape[1]):
            j_prime = j * strides[1]
            for k_row in range(W.shape[0]):
                for k_col in range(W.shape[1]):
                    output[i_prime+k_row, j_prime+k_col] += W[k_row, k_col] * X[i, j]
    return output

def convolute_full_backward(X, dZ, dW, strides=(1, 1)):
    for i in range(0, X.shape[0]):
        i_prime = i * strides[0]
        for j in range(0, X.shape[1]):
            j_prime = j * strides[1]
            for k_row in range(dW.shape[0]):
                for k_col in range(dW.shape[1]):
                    dW[k_row, k_col] += dZ[i_prime+k_row, j_prime+k_col] * X[i, j]
    return dW

def get_errors(predicted, label):
    return label - predicted

def fcn_forward(W, X):
    rotated = np.rot90(np.rot90(W))
    output = convolute_full(X, rotated)
    return output

def fcn_backward(X, dZ, kernel):
    dw = np.zeros(kernel.shape)
    dw = convolute_full_backward(X, dZ, dw)
    dw = np.rot90(np.rot90(dw))
    dx = convolute(dZ, np.rot90(np.rot90(kernel)))
    np.clip(dx, 10, -10)
    return dx, dw

def forward(weights, X):
    values = []
    values.append(X)
    predicted = fcn_forward(weights[0], X)
    values.append(predicted)
    predicted = fcn_forward(weights[1], predicted)
    values.append(predicted)
    return values

def backward(weights, values, label, learningRate=0.001):
    dz = get_errors(values[-1], label)
    dx, dw = fcn_backward(values[-2], dz, weights[-1])
    weights[-1] = weights[-1] + learningRate*dw
    dz = dx
    dx, dw = fcn_backward(values[-3], dz, weights[-2])
    #new apply dw:
    weights[-2] = weights[-2] + learningRate*dw
    return weights

def train_example():
    epoch = int(input("please enter epoch: "))
    inp = np.random.randn(10,10)
    weights = [np.random.randn(3,3), np.random.randn(3,3)]
    label = np.random.randn(14,14)
    for i in range(0, epoch):
        values = forward(weights, inp)
        errors = get_errors(values[-1], label)
        print("error sum at {} is: {}".format(i, np.sum(errors)))
        weights = backward(weights, values, label)

    errors = get_errors(values[-1], label)
    print("error sum at end of training: ", np.sum(errors))

Output:
since your implementation has every scalar multiplication performed explicitly, the backward step can be done very clear. You keep all the loops unchanged, and when you see an update to your accumulator you compute the gradient for that.
import numpy as np
def convolute_full(X, W, strides=(1, 1)):
    row_num = (X.shape[0] - 1) * strides[0] + W.shape[0]
    col_num = (X.shape[1] - 1) * strides[1] + W.shape[1]
    output = np.zeros([row_num, col_num])
    for i in range(0, X.shape[0]):
        i_prime = i * strides[0]
        for j in range(0, X.shape[1]):
            j_prime = j * strides[1]
            for k_row in range(W.shape[0]):
                for k_col in range(W.shape[1]):
                    output[i_prime+k_row, j_prime+k_col] += W[k_row, k_col] * X[i, j]
    return output

def convolute_full_backward(X, dZ, dW, strides=(1, 1)):
    row_num = (X.shape[0] - 1) * strides[0] + W.shape[0]
    col_num = (X.shape[1] - 1) * strides[1] + W.shape[1]
    output = np.zeros([row_num, col_num])
    for i in range(0, X.shape[0]):
        i_prime = i * strides[0]
        for j in range(0, X.shape[1]):
            j_prime = j * strides[1]
            for k_row in range(W.shape[0]):
                for k_col in range(W.shape[1]):
                    # Only this line changed compard to forward pass
                    dW[k_row, k_col] += dZ[i_prime+k_row, j_prime+k_col] * X[i, j]

def fcn_forward(X, W):
    output = convolute_full(X, W[::-1,::-1])
    return output

def fcn_backward(X, dZ, kernel_shape):
    dW = np.zeros(kernel_shape)
    convolute_full_backward(X, dZ, dW[::-1,::-1])
    return dW

To validate I created a simple example with  a linear loss function
X = np.random.randn(20, 20)
W = np.random.randn(5, 5)
Z = fcn_forward(X, W)
# pick a random loss with known gradient
dZ = np.random.randn(*Z.shape)
F = np.sum(Z * dZ)

dW = fcn_backward(X, dZ, W.shape)
# random perturbation
W_ = W + np.random.randn(*W.shape)
# expected change to the loss function
dF = np.sum(dW * (W_ - W))
Z_ = fcn_forward(X, W_)
F_ = np.sum(Z_ * dZ)
print('Predicted loss change: %f' % dF)
print('Actual loss change: %f' % (F_ - F))

Run and see.


Instruction: Use torch.gather to select images from tensor

Input:
I have a tensor of images of size (3600, 32, 32, 3) and I have a multi hot tensor [0, 1, 1, 0, ...] of size (3600, 1). I am looking to basically selecting images that correspond to a 1 in the multi hot tensor. I am trying to understand how to use torch.gather:
tensorA.gather(0, tensorB)
Gives me issues with dims and I can't properly understand how to reshape them.

Output:
When using torch.gather, the dimension of input and dimension of index must be the same. And the index is not a multi hot tensor, but the location of the desired value.
You can slice the tensor by using the index of the multi hot tensor. The fourth line finds the index with a value of 1 in the multi hot tensor. The fifth line slices the image based on the index.
Code:
tensorA = torch.randn(4, 32, 32, 3)
tensorB = torch.tensor([0,1,1,0])

tensorB_where = torch.where(tensorB == 1)[0]
result = tensorA[tensorB_where]


Instruction: Multiple arithmetic operations on single tensor

Input:
For a 2D tensor with even and odd numbers, I would like to
do different arithmetic operation depending on if the number
in the tensor is even or odd.  I've created the tensor and created
a corresponding true false (even or odd) tensor, but am not sure how to proceed.
import torch

list1 = [
[10, 25, 75, 10, 50],
[25, 30, 35, 40, 30],
[45, 50, 55, 60, 20],
[50, 20, 15, 20, 10],
[10, 25, 40, 50, 35]]

tensor2 = torch.tensor(list1)
tensor3=tensor2 % 2
print(tensor3)
print(torch.eq(tensor3, 0)) #even numbers
print(torch.eq(tensor3, 1)) #odd numbers
#do 3x+1 for odd numbers, ie the tensor where indexes are false for even
#do x/2 for even numbers, ie the tensor where indexes are true for even

Output:
You're looking for torch.where
In your case the code would be:
list1 = [
[10, 25, 75, 10, 50],
[25, 30, 35, 40, 30],
[45, 50, 55, 60, 20],
[50, 20, 15, 20, 10],
[10, 25, 40, 50, 35]]

tensor2 = torch.tensor(list1, type = torch.float64)
res = torch.where(tensor2 % 2 == 0, tensor2 / 2, tensor2 * 3 + 1)

>>> res
tensor([[  5.,  76., 226.,   5.,  25.],
        [ 76.,  15., 106.,  20.,  15.],
        [136.,  25., 166.,  30.,  10.],
        [ 25.,  10.,  46.,  10.,   5.],
        [  5.,  76.,  20.,  25., 106.]], dtype=torch.float64)


Instruction: Convert multirelational graphs to bipartite graphs with networkx

Input:
I have a multi-relational graph G (subject s and object o nodes connected with an edge having a predicate label p), made with networkx.DiGraph. Is there a way to easily and elegantly create its bipartite graph by replacing each labeled edge (s, p, o) with two unlabeled edges {(s,p),(p,o)}? This operation is also called Levi transformation (Levi, 1942) and is aimed to treat each entity and relation equally.
Current graph construction example:
amr_graph = networkx.DiGraph()

for node_id, node_string in amr.nodes.items():
   amr_graph.add_node(node_id, name=node_string)

for subj, pred, obj in amr.edges:
   amr_graph.add_edge(subj, obj, key=pred)

Output:
Just create the new edges as you see fit:
import random
import networkx as nx

s=[random.randint(0,2) for i in range(10)]
o=[random.randint(3,5) for i in range(10)]
p=[random.randint(6,8) for i in range(10)]

G = nx.DiGraph()
H = nx.DiGraph()
F = nx.DiGraph()

#split while constructing the orgiginal graph
for subj, pred, obj in zip(s,p,o):
    G.add_edge(subj, obj, key= pred)

    H.add_edge(subj, pred, key = obj)
    H.add_edge(pred, obj, key= subj)

#use existing graph to split edges
for subj, obj, d in G.edges(data=True):
    pred = d['key']

    F.add_edge(subj, pred, key = obj)
    F.add_edge(pred, obj, key= subj)


Instruction: "Extracting" the indices tensor from sparse torch tensor

Input:
Given a torch tensor A with sparse entries (say, for example, the adjacency matrix of a graph), we can construct a sparse tensor from A by using the function to_sparse().
For example:
A_sparse = A.to_sparse()
>>> tensor(indices=tensor([[   0,    0,    0,  ..., 2707, 2707, 2707],
                       [ 633, 1862, 2582,  ...,  598, 1473, 2706]]),
       values=tensor([1., 1., 1.,  ..., 1., 1., 1.]),
       size=(2708, 2708), nnz=10556, layout=torch.sparse_coo)

But note that:
A_sparse.shape
>>> torch.Size([2708, 2708])

What I would like is to only obtain the indices tensor in the above "tuple", that is to say a tensor of shape (2,10556).
I have tried A_sparse[0]and A_sparce[indices] but neither method returns the expected result.

Output: Following skyzip's comment, it is sufficient to call A_sparse.indices()


Instruction: RuntimeError: Failed to import transformers.pipelines because of the following error (look up to see its traceback): initialization failed

Input:
Goal: Run a GPT-2 model instance.
I am using the latest Tensorflow and Hugging Face  Transformers.

Tensorflow - 2.9.1
Transformers - 4.21.1

Notebook:
pip install tensorflow

pip install transformers

from transformers import pipeline, set_seed

generator = pipeline('text-generation', model='gpt2')
set_seed(42)

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
RuntimeError: module compiled against API version 0xe but this version of numpy is 0xd

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
ImportError: numpy.core.multiarray failed to import

The above exception was the direct cause of the following exception:

SystemError                               Traceback (most recent call last)
SystemError: <built-in method __contains__ of dict object at 0x7f5b58a64d00> returned a result with an error set

The above exception was the direct cause of the following exception:

ImportError                               Traceback (most recent call last)
~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/utils/import_utils.py in _get_module(self, module_name)
   1001         try:
-> 1002             return importlib.import_module("." + module_name, self.__name__)
   1003         except Exception as e:

~/anaconda3/envs/python3/lib/python3.8/importlib/__init__.py in import_module(name, package)
    126             level += 1
--> 127     return _bootstrap._gcd_import(name[level:], package, level)
    128

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _gcd_import(name, package, level)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _find_and_load(name, import_)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _find_and_load_unlocked(name, import_)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _load_unlocked(spec)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap_external.py in exec_module(self, module)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/pipelines/__init__.py in <module>
     36 from ..utils import HUGGINGFACE_CO_RESOLVE_ENDPOINT, http_get, is_tf_available, is_torch_available, logging
---> 37 from .audio_classification import AudioClassificationPipeline
     38 from .automatic_speech_recognition import AutomaticSpeechRecognitionPipeline

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/pipelines/audio_classification.py in <module>
     19 from ..utils import add_end_docstrings, is_torch_available, logging
---> 20 from .base import PIPELINE_INIT_ARGS, Pipeline
     21

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/pipelines/base.py in <module>
     33 from ..feature_extraction_utils import PreTrainedFeatureExtractor
---> 34 from ..modelcard import ModelCard
     35 from ..models.auto.configuration_auto import AutoConfig

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/modelcard.py in <module>
     43 )
---> 44 from .training_args import ParallelMode
     45 from .utils import (

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/training_args.py in <module>
     25 from .debug_utils import DebugOption
---> 26 from .trainer_utils import (
     27     EvaluationStrategy,

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/trainer_utils.py in <module>
     46 if is_tf_available():
---> 47     import tensorflow as tf
     48

~/anaconda3/envs/python3/lib/python3.8/site-packages/tensorflow/__init__.py in <module>
     36
---> 37 from tensorflow.python.tools import module_util as _module_util
     38 from tensorflow.python.util.lazy_loader import LazyLoader as _LazyLoader

~/anaconda3/envs/python3/lib/python3.8/site-packages/tensorflow/python/__init__.py in <module>
     36 from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow
---> 37 from tensorflow.python.eager import context
     38

~/anaconda3/envs/python3/lib/python3.8/site-packages/tensorflow/python/eager/context.py in <module>
     34 from tensorflow.python import tf2
---> 35 from tensorflow.python.client import pywrap_tf_session
     36 from tensorflow.python.eager import executor

~/anaconda3/envs/python3/lib/python3.8/site-packages/tensorflow/python/client/pywrap_tf_session.py in <module>
     18 from tensorflow.python import pywrap_tensorflow
---> 19 from tensorflow.python.client._pywrap_tf_session import *
     20 from tensorflow.python.client._pywrap_tf_session import _TF_SetTarget

ImportError: initialization failed

The above exception was the direct cause of the following exception:

RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_4924/2487422996.py in <cell line: 1>()
----> 1 from transformers import pipeline, set_seed
      2
      3 generator = pipeline('text-generation', model='gpt2')
      4 set_seed(42)

~/anaconda3/envs/python3/lib/python3.8/importlib/_bootstrap.py in _handle_fromlist(module, fromlist, import_, recursive)

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/utils/import_utils.py in __getattr__(self, name)
    990             value = self._get_module(name)
    991         elif name in self._class_to_module.keys():
--> 992             module = self._get_module(self._class_to_module[name])
    993             value = getattr(module, name)
    994         else:

~/anaconda3/envs/python3/lib/python3.8/site-packages/transformers/utils/import_utils.py in _get_module(self, module_name)
   1002             return importlib.import_module("." + module_name, self.__name__)
   1003         except Exception as e:
-> 1004             raise RuntimeError(
   1005                 f"Failed to import {self.__name__}.{module_name} because of the following error (look up to see its"
   1006                 f" traceback):\n{e}"

RuntimeError: Failed to import transformers.pipelines because of the following error (look up to see its traceback):
initialization failed

def query(payload, multiple, min_tokens, max_tokens):
    nlp_setup()
    list_dict = generator(payload, min_length=min_tokens, max_new_tokens=max_tokens, num_return_sequences=multiple)
    return [d['generated_text'].split(payload)[1].strip() for d in list_dict

output = query("Banking customer's needs:", 3000, 50, 50)

RunTime, SystemError and ImportError all occur during import of transformers:
RuntimeError: Failed to import transformers.pipelines because of the following error (look up to see its traceback): initialization failed

Output: Changed Kernel: conda_tensorflow2_p38


Instruction: Check if tensor is in list

Input:
I can do the following with a single int to retrieve a bool tensor:
import torch
a = torch.tensor([1,2,3])
a != 2
#tensor([ True, False,  True])

Can I do the same with a list in plain pytorch? I.e.:
import torch
a = torch.tensor([1,2,3])
a not in [2,3]
#tensor([ True, False,  False])

Thanks a lot for your time!

Output:
I think you want torch.isin
out = ~torch.isin(a, torch.tensor([2, 3]))
# or
out = torch.isin(a, torch.tensor([2, 3]), invert=True)

print(out)

tensor([ True, False, False])


Instruction: How do I tie embeddings between a `torch.nn.Embedding` and `torch.nn.EmbeddingBag`?

Input:
I'd like to tie the embedding layers between two parts of my neural network: one which embeds tokens where order matters (i.e., nn.Embedding) and one which embeds tokens where order doesn't matter (i.e., nn.EmbeddingBag). I ran into numerical stability issues when creating my own EmbeddingBag-like object and doing the reduction myself so I'd like to use the officially support nn.EmbeddingBag; however, it seems like my attempt to tie weights (below) doesn't work
#!/usr/bin/env python3

import torch
import torch.nn as nn

if __name__ == "__main__":
    V, max_seq, padding_idx, emb_dim, B = 10, 100, 1, 512, 32

    # Create an embedding layer and initialize an embeddingbag with those weights
    emb_layer = nn.Embedding(V, emb_dim, padding_idx=padding_idx)
    emb_bag = nn.EmbeddingBag.from_pretrained(emb_layer.weight, freeze=False, padding_idx=padding_idx)

    tokens = torch.randint(0, V, (B, max_seq))
    y = torch.randn((B, emb_dim))
    loss = nn.MSELoss()

    # backprop through the embedding bag
    y_ = emb_bag(tokens)
    l = loss(y_, y)
    assert emb_bag.weight.grad is None
    l.backward()
    assert emb_bag.weight.grad is not None

    # if we're tying weights, backpropping through the emb bag should
    # yield the same gradients in the embedding layer, but... the following assertion fails
    assert emb_layer.weight.grad is not None and \
        torch.allclose(emb_bag.weight.grad, emb_layer.weight.grad)

Is there some way to tie the weights in both or do I need to be creative with how I emulate the embeddingbag behavior?

Output:
from_pretrained() will copy the weights: emb_layer.weight is emb_bag.weight will be False.
You can just set the weight attribute directly:
emb_bag.weight = emb_layer.weight


Instruction: PyTorch: CUDA is not available

Input:
I'm trying to run PyTorch on a NVIDIA Jetson Nano and my project requires me to use CUDA. I'm running on Ubuntu 18.04 and Python 3.10.6. I followed this guide to install CUDA 11.6. Then using the instructions on PyTorch.org I installed PyTorch using this command:
pip install torch==1.12.0 torchvision==0.13.0 --extra-index-url https://download.pytorch.org/whl/cu116

But then when I try to verify it, it's not available:
(env) $ python
Python 3.10.6 (main, Aug  2 2022, 15:11:03) [GCC 7.5.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.cuda.is_available()
False
>>>

here's the CUDA version:
(env) $ nvcc --version
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2021 NVIDIA Corporation
Built on Fri_Dec_17_18:16:35_PST_2021
Cuda compilation tools, release 11.6, V11.6.55
Build cuda_11.6.r11.6/compiler.30794723_0

And here is some information from PyTorch. Notice how it says CUDA used to build PyTorch: Could not collect.
(env) $ python -m torch.utils.collect_env
PyTorch version: 1.12.0
Is debug build: False
CUDA used to build PyTorch: Could not collect
ROCM used to build PyTorch: N/A

OS: Ubuntu 18.04.6 LTS (aarch64)
GCC version: (Ubuntu/Linaro 7.5.0-3ubuntu1~18.04) 7.5.0
Clang version: Could not collect
CMake version: version 3.10.2
Libc version: glibc-2.27

Python version: 3.10.6 (main, Aug  2 2022, 15:11:03) [GCC 7.5.0] (64-bit runtime)
Python platform: Linux-4.9.253-tegra-aarch64-with-glibc2.27
Is CUDA available: False
CUDA runtime version: 11.6.55
GPU models and configuration: Could not collect
Nvidia driver version: Could not collect
cuDNN version: Probably one of the following:
/usr/lib/aarch64-linux-gnu/libcudnn.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_adv_infer.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_adv_train.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_cnn_infer.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_cnn_train.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_ops_infer.so.8.2.1
/usr/lib/aarch64-linux-gnu/libcudnn_ops_train.so.8.2.1
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

Versions of relevant libraries:
[pip3] numpy==1.23.1
[pip3] torch==1.12.0
[pip3] torchvision==0.13.0
[conda] Could not collect

Any help is appreciated. Thanks.

Output:
Downloading the compiled whl file from nvidia and installing it with pip did the trick (as suggested by @FlyingTeller).
wget https://nvidia.box.com/shared/static/p57jwntv436lfrd78inwl7iml6p13fzh.whl -O torch-1.8.0-cp36-cp36m-linux_aarch64.whl
pip install numpy torch-1.8.0-cp36-cp36m-linux_aarch64.whl


Instruction: Convert from RGB to LAB using pytorch? Opencv's color conversion does not match their own formula

Input:
In order to speed up some of my past opencv/numpy powered operations, I am attempting to translate them to PyTorch, to run on GPU. One of the first of these steps it RGB to LAB conversion.
I referred to:

the opencv documentation on their own color conversion
the Wikipedia page on CIE Lab, which offers one version of the algorithm (consistent with opencv's version when accounting for some refactoring and different XYZ starting ranges)
this implementation from EdgeFool, which is already in PyTorch, but expects different value ranges and tensor shapes.

I have managed to produce this code:
import torch

def tensor_like(source_data, target_tensor):
    return torch.tensor(
        source_data,
        device=target_tensor.device,
        dtype=target_tensor.dtype,
    )

RGB_TO_XYZ_FACTOR_MATRIX = [
    [0.412453, 0.357580, 0.180423],
    [0.212671, 0.715160, 0.072169],
    [0.019334, 0.119193, 0.950227],
]

def cie_f(input_: torch.Tensor):
    e = 6 / 29
    return torch.where(
        input_ > e ** 3,
        input_ ** (1 / 3),
        input_ / (3 * e ** 2) + 4 / 29,
    )

def tensor_rgb_to_lab(input_: torch.Tensor):
    # input_ is expected to be RGB 0-255, and of shape B, C, H, W
    # Implemented based on formulas written here (may need to scroll down):
    # https://docs.opencv.org/4.x/de/d25/imgproc_color_conversions.html
    dtype = input_.dtype
    if not torch.is_floating_point(input_):
        input_ = input_.float()

    input_ = input_.permute(0, 2, 3, 1)
    rgb_colors = input_.reshape(-1, 3)
    rgb_colors /= 255.0

    # RGB -> XYZ
    factor_matrix = tensor_like(RGB_TO_XYZ_FACTOR_MATRIX, rgb_colors)
    xyz_colors = torch.mm(rgb_colors, factor_matrix.T)
    # xyz_colors.mul_(255)

    # XYZ -> LAB
    xyz_colors *= tensor_like([1 / 0.950456, 1.0, 1 / 1.088754], xyz_colors)
    f_xyz_colors = cie_f(xyz_colors)
    fxyz_to_lab_factor_matrix = tensor_like([
        [0.0, 500.0, 0.0],
        [116.0, -500.0, 200.0],
        [0.0, 0.0, -200.0],
    ], f_xyz_colors)
    lab_colors = torch.mm(f_xyz_colors, fxyz_to_lab_factor_matrix)
    lab_colors += tensor_like([-16, 128, 128], lab_colors)
    lab_colors[:, 0] *= 255 / 100

    lab_colors = lab_colors.view_as(input_).permute(0, 3, 1, 2)
    lab_colors = lab_colors.round_().clamp_(0, 255).type(dtype)
    return lab_colors

When comparing my output against cv2.cvtColor(image, cv2.COLOR_RGB2LAB), my code does produce the correct results with RGB colors like (000, 000, 000), (255, 255, 255), (000, 255, 255), (255, 000, 255), (255, 255, 000), (255, 000, 000), (000, 255, 000), (000, 000, 255), but it fails on many other cases (more than 99% of RGB colors, according to my test).
One such failing case is RGB(128, 128, 128), which open-cv says corresponds to Lab(137, 128, 128), but my code indicates Lab(194, 128, 128). This is an interesting failing case since only 1 value is wrong, and I can manually go through the different steps of the formula for L only in an attempt to isolate the problem:
g = np.array((0.412453+0.357580+0.180423, 0.212671+0.715160+0.072169,0.019334+0.119193+0.950227)) * 128/255
g[0]/=0.950456
g[2]/=1.088754
# g => array([0.50196078, 0.50196078, 0.50196078]). Makes sense since 128/255=0.50196078
l = 116*g[1]**(1/3)-16  # since 0.50196078>0.008856
# l => 76.18945600835188  This is already wrong according to online RGB converters
l *= 255/100
# l => 194.2831128212973  This contradicts opencv's value and correlates with mine

After quadruple-checking my code, I was starting to think opencv might have a bug/rounding error in their code, or that they were doing an extra step differently than they say on their documentation. So I have looked up some online RGB to Lab converters (1 and 2), and after re-adjusting the output L value to the 255 range, they agree with opencv. I guess this really points to a mistake on my part (unless the websites internally rely on opencv?).
Considering how the result comes out correct for RGB values of both 0 and 255 (or 0 and 1 when scaled to 0..1), I think that all of the ax+b operations must be correct, and the t**(1/3) must be the issue... could the industry be somehow using a different exponent, maybe due to performance benefit?
What am I missing here?
I am using opencv 4.4.0.46

Output:
Instead of implementing this conversion yourself, you can use kornia's implementation: kornia.color.rgb_to_lab(image) seems to do exactly what you are trying to do in your tensor_rgb_to_lab function.
If you insist, you can checkout their implementation here.


Instruction: pytorch, for the cross_entropy function, What if the input labels are all ignore_index?

Input:
for the nn.CrossEntropyLoss(),  What if the input labels are all ignore_index ? I got a 'nan' loss, how to fix it ? thanks!
detail of my code is following:
code
notice that IGNORE_INDEX is -100
then, the loss_ent becomes 'nan', the input of func criterion() is following:
criterion's input
my python environment is following:

Python:3.7.11
torch:1.12.0+cu116
GPU:NVIDIA A100 80G

thank for your attention, Please let me know if there is anything else I need to provide.

Output:
There seem to be two possibilities.

This is the code to help understand the ignore index. If you set all inputs to ignore index, criterion makes nan as output because there is no value to compute.

import torch
import torch.nn as nn

x = torch.randn(5, 10, requires_grad=True)
y = torch.ones(5).long() * (-100)

criterion = nn.CrossEntropyLoss(ignore_index=-100)
loss = criterion(x, y)
print(loss)

tensor(nan, grad_fn=<NllLossBackward0>)

However, if there is even one value to calculate, the loss is calculated properly.
import torch
import torch.nn as nn

x = torch.randn(5, 10, requires_grad=True)
y = torch.ones(5).long() * (-100)
y[0] = 1

criterion = nn.CrossEntropyLoss(ignore_index=-100)
loss = criterion(x, y)
print(loss)

tensor(1.2483, grad_fn=<NllLossBackward0>)

It seems the learning rate is too large that cause diverge.

As you can see in the figure below, if the learning rate is large, the gradient gradually increases and diverges.
How about reducing the learning rate of the parameters of the model with is_gold_ent as output?
Usually, when reducing the learning rate, reduce it by 1/3. For example 0.01, 0.003, 0.001,...


Instruction: How to set minimum k elements in a dimension in pytorch tensor to a specific value?

Input:
For example, if I have tensor (shape [2, 3, 5])
[[[0.8823, 0.9150, 0.3829, 0.9593, 0.3904],
  [0.6009, 0.2566, 0.7936, 0.9408, 0.1332],
  [0.9346, 0.5936, 0.8694, 0.5677, 0.7411]],

 [[0.4294, 0.8854, 0.5739, 0.2666, 0.6274],
  [0.2696, 0.4414, 0.2969, 0.8317, 0.1053],
  [0.2695, 0.3588, 0.1994, 0.5472, 0.0062]]]

and k = 2, I want to set minimum k elements in a dimension (e.g., dim=2) to a specific value (e.g., 5):
[[[0.8823, 0.9150, 5, 0.9593, 5],
  [0.6009, 5, 0.7936, 0.9408, 5],
  [0.9346, 5, 0.8694, 5, 0.7411]],

 [[5, 0.8854, 0.5739, 5, 0.6274],
  [5, 0.4414, 0.2969, 0.8317, 5],
  [0.2695, 0.3588, 5, 0.5472, 5]]]

Output:
You can use a combination of torch.topk and torch.Tensor.scatter_.
(Bacause torch.topk return max_top_k and you want min_top_k. We can use -1*all_num for getting min_top_k)
val, ind = torch.topk(-a, k=2)
a.scatter_(index=ind, dim=-1, value=5)
print(a)

tensor([[[0.8823, 0.9150, 5.0000, 0.9593, 5.0000],
         [0.6009, 5.0000, 0.7936, 0.9408, 5.0000],
         [0.9346, 5.0000, 0.8694, 5.0000, 0.7411]],

        [[5.0000, 0.8854, 0.5739, 5.0000, 0.6274],
         [5.0000, 0.4414, 0.2969, 0.8317, 5.0000],
         [0.2695, 0.3588, 5.0000, 0.5472, 5.0000]]])

Input:
>>> a = torch.tensor([[[0.8823, 0.9150, 0.3829, 0.9593, 0.3904],
                      [0.6009, 0.2566, 0.7936, 0.9408, 0.1332],
                      [0.9346, 0.5936, 0.8694, 0.5677, 0.7411]],

                     [[0.4294, 0.8854, 0.5739, 0.2666, 0.6274],
                      [0.2696, 0.4414, 0.2969, 0.8317, 0.1053],
                      [0.2695, 0.3588, 0.1994, 0.5472, 0.0062]]])

>>> torch.topk(-a, k=2)
# values=tensor(
#     [[[-0.3829, -0.3904],
#       [-0.1332, -0.2566],
#       [-0.5677, -0.5936]],

#      [[-0.2666, -0.4294],
#       [-0.1053, -0.2696],
#       [-0.0062, -0.1994]]]),

# indices=tensor(
#     [[[2, 4],
#       [4, 1],
#       [3, 1]],

#      [[3, 0],
#       [4, 0],
#       [4, 2]]])


Instruction: How to return torch model weights from endpoint Flask

Input:
I'm developing a project in PyTorch which requires returning the weights (state_dict) of a pytorch model as a Flask endpoint response. To better explain it, the simplest code could be:
@app.endpoint('/send_weights', methods=['GET', 'POST'])
def send_weights():
    model_weights = model.state_dict() # It is a dict[str, torch.tensor]
    return model_weights

However it is not as simple because the torch.tensor is not JSON serializable, so, I've tried to convert them to a list (JSON serializable object) and it works:
@app.endpoint('/send_weights', methods=['GET', 'POST'])
def send_weights():
    model_weights = model.state_dict() # It is a dict[str, torch.tensor]
    model_weights = {k:v.tolist() for k,v in model_weights.items()}
    return model_weights

However this process is very slow and it doesn't meet my requirements. I was trying to convert the tensors to bytes but the code gives the same problem, bytes is not JSON serializable. So, I'm thinking that the json response won't be the solution.
I'm not an expert in Flask but I've read about the flask send_file method, however I not sure how to use it in this case (not even sure this would be a possible solution), I haven't got a mimetype for the dictionary.
Does anybody know a better way to do this?

Output:
I've just found a solution. It is based on return the weights as a file using the send_file method from flask with the weights saved on a binary using torch.save and the mimetype 'application/octet-stream' as appears in this question.
The final endopint code will be:
@app.endpoint('/send_weights', methods=['GET', 'POST'])
def send_weights():
    model_weights = model.state_dict()
    to_send = io.BytesIO()
    torch.save(model_weights, to_send, _use_new_zipfile_serialization=False)
    to_send.seek(0)
    return send_file(to_send, mimetype='application/octet-stream')

And, to load it in the other side:
weights = torch.load(io.BytesIO(response.content))

Hope it will help somebody.


Instruction: Multiple input model with PyTorch - input and output features

Input:
Good afternoon!
I’m building a multiple-input model with 2 types of inputs:
Images (torch.Size([1, 3, 224, 224])) and landmark features (torch.Size([1, 96])).
Here’s the model itself:
class MixedNetwork(nn.Module):
    def __init__(self):
        super(MixedNetwork, self).__init__()

        image_modules = list(models.resnet50().children())[:-1]
        self.image_features = nn.Sequential(*image_modules)

        self.landmark_features = nn.Sequential(
            nn.Linear(in_features=96, out_features=192,bias=False),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.25),
            nn.Linear(in_features=192,out_features=1000,bias=False),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.25))

        self.combined_features = nn.Sequential(
            nn.Linear(1000, 512),
            nn.ReLU(),
            nn.Linear(512, 32),
            nn.ReLU(),
            nn.Linear(32,1))

    def forward(self, image, landmarks):
        a = self.image_features(image)
        print(a.shape)
        b = self.landmark_features(landmarks)
        x = torch.cat((a.view(a.size(0), -1), b.view(b.size(0), -1)), dim=1)
        x = self.combined_features(x)
        x = F.sigmoid(x)
        return x

I’m getting confused when it comes to defining input-output features for Linear layers and combined layers. The last FC layer of resnet50 Linear(in_features=2048, out_features=1000). Does it mean that the last output of self.landmark_features layers also has to be 1000 and the first linear layer of self.combined_features should also be 1000?
Is it correct to assume that if the landmark input size is [1, 96] then the in_features for the first layer of self.landmark_features has to be 96?
With the current dimensions I’m getting the error message:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x3048 and 1000x512)
(why 3048 and not 2048?)

Output:
This code makes image shape(1, 3, 224, 224) to (1, 2048)
a = self.image_features(image)
a = a.view(a.size(0), -1)

And this code makes landmark features shape(1, 96) to (1,1000)
b = self.landmark_features(landmarks)
a = b.view(b.size(0), -1)

And torch cat makes concatenate a, b vectors. So, self.combined_features(x) in x's shape is (1, 3048)
So, the code should be changed to the following:
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.models as models
import torch.nn.functional as F

class MixedNetwork(nn.Module):
    def __init__(self):
        super(MixedNetwork, self).__init__()

        image_modules = list(models.resnet50().children())[:-1]
        self.image_features = nn.Sequential(*image_modules)

        self.landmark_features = nn.Sequential(
            nn.Linear(in_features=96, out_features=192,bias=False),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.25),
            nn.Linear(in_features=192,out_features=1000,bias=False),
            nn.ReLU(inplace=True),
            nn.Dropout(p=0.25))

        self.combined_features = nn.Sequential(
            # change this input nodes
            nn.Linear(3048, 512),
            nn.ReLU(),
            nn.Linear(512, 32),
            nn.ReLU(),
            nn.Linear(32,1))

    def forward(self, image, landmarks):
        a = self.image_features(image)
        b = self.landmark_features(landmarks)
        x = torch.cat((a.view(a.size(0), -1), b.view(b.size(0), -1)), dim=1)
        x = self.combined_features(x)
        x = torch.sigmoid(x)
        return x

model = MixedNetwork()
batch_size = 1

# random input
image = torch.randn(batch_size, 3, 224, 224)
land = torch.randn(batch_size, 96)

output = model(image, land)
print(output)


Instruction: How to train model with multiple GPUs in pytorch？

Input:
My server has two GPUs, How can I use two GPUs for training at the same time to maximize their computing power? Is my code below correct? Does it allow my model to be properly trained?
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.bert = pretrained_model
        # for param in self.bert.parameters():
        #     param.requires_grad = True
        self.linear = nn.Linear(2048, 4)

    #def forward(self, input_ids, token_type_ids, attention_mask):
    def forward(self, input_ids, attention_mask):
        batch = input_ids.size(0)
        #output = self.bert(input_ids, token_type_ids, attention_mask).pooler_output
        output = self.bert(input_ids, attention_mask).last_hidden_state
        print('last_hidden_state',output.shape) # torch.Size([1, 768])
        #output = output.view(batch, -1) #
        output = output[:,-1,:]#(batch_size, hidden_size*2)(batch_size,1024)
        output = self.linear(output)
        return output

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
    print("Use", torch.cuda.device_count(), 'gpus')
    model = MyModel()
    model = nn.DataParallel(model)
    model = model.to(device)

Output:
There are two different ways to train on multiple GPUs:

Data Parallelism  = splitting a large batch that can't fit into a single GPU memory into multiple GPUs, so every GPU will process a small batch that can fit into its GPU
Model Parallelism = splitting the layers within the model into different devices is a bit tricky to manage and deal with.

Please refer to this post for more information
To do Data Parallelism in pure PyTorch, please refer to this example that I created a while back to the latest changes of PyTorch (as of today, 1.12).
To utilize other libraries to do multi-GPU training without engineering many things, I would suggest using PyTorch Lightning as it has a straightforward API and good documentation to learn how to do multi-GPU training using Data Parallelism.

Update: 2022/10/25
Here is a video explaining in much details about different types of distributed training: https://youtu.be/BPYOsDCZbno?t=1011


Instruction: How do I convert a torch tensor to an image to be returned by FastAPI?

Input:
I have a torch tensor which I need to convert to a byte object so that I can pass it to starlette's StreamingResponse which will return a reconstructed image from the byte object. I am trying to convert the tensor and return it like so:
def some_unimportant_function(params):
    return_image = io.BytesIO()
    torch.save(some_img, return_image)
    return_image.seek(0)
    return_img = return_image.read()

    return StreamingResponse(content=return_img, media_type="image/jpeg")

The below works fine on regular byte objects and my API returns the reconstructed image:
def some_unimportant_function(params):
    image = Image.open(io.BytesIO(some_image))

    return_image = io.BytesIO()
    image.save(return_image, "JPEG")
    return_image.seek(0)
    return StreamingResponse(content=return_image, media_type="image/jpeg")

Using PIL library for this
what am I doing wrong here?

Output:
Converting PyTorch Tensor to the PIL Image object using torchvision.transforms.ToPILImage() module and then treating it as PIL Image as your second function would work. Here is an example.
def some_unimportant_function(params):
    tensor = # read the tensor from disk or whatever
    image = torchvision.transforms.ToPILImage()(tensor.unsqueeze(0))
    return_image = io.BytesIO()
    image.save(return_image, "JPEG")
    return_image.seek(0)
    return StreamingResponse(content=return_image, media_type="image/jpeg")


Instruction: Why is `nn.Unfold` not giving me the same results as `.unfold`

Input:
Why is this giving me two completely different answers? And how can I get the same result as in approach 1, using approach 2?
import torch
from torch import nn

kernel_size = 7
stride = 1

# approach 1
data = torch.rand(4, 64, 174, 120)
data1 = data.unfold(3, kernel_size * 2 + 1, stride)
print(data1.shape)

# approach 2
data = torch.rand(4, 64, 174, 120)
unfold = nn.Unfold(3, kernel_size * 2 + 1, stride)
data2 = unfold(data)
print(data2.shape)

Output:
torch.Size([4, 64, 174, 106, 15])
torch.Size([4, 576, 13432])

EDIT ------------------------------------------
I tried your approach @Shai. The shapes are the same, but the content is not. Any idea why?
import torch
from torch import nn

kernel_size = 7
stride = 1

# approach 1
data = torch.rand(4, 64, 174, 120)
data1 = data.unfold(3, kernel_size * 2 + 1, stride)
print(data1.shape)

# approach 2
data = torch.rand(4, 64, 174, 120)
b, c, h, w = data.shape
unfold = nn.Unfold(kernel_size=(1, 2*kernel_size + 1), dilation=1, stride=1, padding=0)
data2 = unfold(data.reshape(-1, 1, 1, w)).permute(0, 2, 1).reshape(b, c, h, -1, 2*kernel_size + 1)
print(data2.shape)

print(torch.equal(data1, data2))

Output:
torch.Size([4, 64, 174, 106, 15])
torch.Size([4, 64, 174, 106, 15])
False

Output:
torch.unfold "unfolds" along a certain dimension. In your example it takes 4x64x174 samples of dim 120 and extract all overlapping 15-windows, resulting with data1 of shape 4x64x174x106x15.
In contrast, nn.Unfold works on bxcx... tensors and extract spatial patches. In your example, nn.Unfold got kernel_size=3, dilation=kernel_size*2+1 and padding=1. Hence, it extracted 13,432 3x3 patches of 64 channels (3364=576), resulting with data2 of shape 4x576x13432.
To get the same output of torch.unfold from nn.Unfold you need to reshape and permute:
b, c, h, w = data.shape
unfold = nn.Unfold(kernel_size=(1, 2*kernel_size + 1), dilation=1, stride=1, padding=0)
data2 = unfold(data.reshape(-1, 1, 1, w)).permute(0, 2, 1).reshape(b, c, h, -1, 2*kernel_size + 1)

Please read carefully the doc of nn.Unfold as it works in a fundamentally different manner to torch.unfold. For more information about nn.Unfold and nn.Fold please see this thread.


Instruction: how to pad a text after build the vocab in pytorch

Input:
I used torchtext vocab to convert the text to index, but which function should I use to make all the index list be the same length before I send them to the net?
For example I have 2 texts:
I am a good man
I would like a coffee please

After vocab:
[1, 3, 2, 5, 7]
[1, 9, 6, 2, 4, 8]

And what I want is:
[1, 3, 2, 5, 7, 0]
[1, 9, 6, 2, 4, 8]

Output:
It is easy to understand by looking at the following example.
Code:
import torch

v = [
        [0,2],
        [0,1,2],
        [3,3,3,3]
]

torch.nn.utils.rnn.pad_sequence([torch.tensor(p) for p in v], batch_first=True)

Result:
tensor([[0, 2, 0, 0],
        [0, 1, 2, 0],
        [3, 3, 3, 3]])


Instruction: Mapping 3D tensor according to a rank tensor

Input:
I have a tensor called rank which is of shape 2x3 as follows:
tensor([[ 0, 1,  2],
        [ 2,  0,  1])

I want to construct a 2X3x3 matrix where the inner matrix is populated initially to all zeros using torch.zeros(2,3,3). I want to update the last dimension to value 1 for last dimension indices corresponding to the values in rank tensor. using indices given in rank.
Final output :
    tensor([
           tensor([[1, 0,  0],
                   [0, 1, 0],
                   [0, 0, 1]],
           tensor([[[0, 0, 1],
                   [1, 0, 0],
                   [0, 1,  0]]
           ])

The value 1 is populated according to the rank given in the rank tensor. How can I do this in in pytorch and python.

Output:
Given:
i=torch.tensor([[ 0, 1, 2],
                [ 2, 0, 1]])

x=torch.tensor([[[1, 0,  0],
                 [0, 1, 0],
                 [0, 0, 1]],

                [[0, 0, 1],
                 [1, 0, 0],
                 [0, 1,  0]]])

You can perform this operation using torch.scatter_:
>>> torch.zeros(2,3,3).scatter_(2, i[:,:,None].expand_as(x), value=1)


Instruction: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead

Input:
I started getting the following warning repeatedly when running my PyTorch Lightning deep learning scripts, at execution start and then all through the training:

"OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead."

I get them when executing the main.py script; my scripts are publicly available here.
Symptoms:

I don't think it has anything to do with PyTorch Lightning, maybe even PyTorch.
It appeared overnight, so I don't know what could cause it.
It runs fine without those warnings on my PC.
I get the warnings when I run from my M1 Mac.
I use VSCode for both, each up to date.
I use separate miniconda environments.

Output:
Thanks for taking the time to reply!
I managed to sort myself out in the end.
I spotted the numba package in my miniconda env, which is a Python compiler and that seemed to be the root of the problem.
It was version 0.55.2 but the last version to date is 0.56.0. Trying to upgrade it via conda or pip didn't work for some reason (the 0.55.2 version couldn't be replaced).
I recreated my env step by step, and noticed this package comes with torch-audiomentations, a package for audio data augmentation for deep learning, under torch, that I use.
Re-installing it had numba version 0.56.0 installed properly, and the warnings disappeared.


Instruction: How to create a dataset containing one datapoint and its augmentations?

Input:
I have an imagenet dataset. I want to extract only one image of the dataset and apply random augmentations on this single data point. So basically the transform I define is as following:
sample_transform = transforms.Compose([
        transforms.RandomResizedCrop(img_size, scale=(0.2, 1.0)),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean=_IMAGENET_RGB_MEANS, std=_IMAGENET_RGB_STDS),
    ])

And assume that my one sample is as follows:
from torch.utils.data import Subset
def extract_sample(data, idx):
    return Subset(data, [idx])
single_sample = extract_sample(data, idx = 0)

Not I want to create a dataset by applying sample_transform, 64 times on single_sample (in order to get 64 different augmentations of single_sample). How can I do that?
Thanks in advance for your help!

Output:
I'd utilize torch.utils.data.RandomSampler with replacement:
import torch
from torchvision import transforms
from torchvision.datasets import ImageFolder
from torch.utils.data import Subset, DataLoader, RandomSampler

import matplotlib.pyplot as plt
from PIL import Image

sample_transform = transforms.Compose([
        transforms.RandomResizedCrop(224, scale=(0.2, 1.0)),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(0.5, 0.5),
    ])

data = ImageFolder('/media/storage/datasets/t', transform=sample_transform)

def extract_sample(data, idx):
    return Subset(data, [idx])

single_sample = extract_sample(data, idx = 0)
sampler = RandomSampler(single_sample, num_samples=64, replacement=True)
loader = DataLoader(single_sample, sampler=sampler)

for img, target in loader:
    plt.imshow(img.squeeze().permute(1, 2, 0))
    plt.show()


Instruction: Learning multivariate normal covariance matrix using pytorch

Input:
I am trying to learn a multivariate normal covariance matrix (Sigma, ∑) using some observations.
The way I went at it is by using pytorch.distributions.MultivariateNormal:
import torch
from torch.distributions import MultivariateNormal

# I tried both the scale_tril parameter and the covariance parameter.
mvn = MultivariateNormal(loc=torch.tensor([0.0, 0.0], requires_grad=False).view(1,2),
                        scale_tril=torch.tensor([[1.0 , 0.0], [0.0, 1.0]],
                                                requires_grad=True).view(-1, 2, 2))

loss = -mvn.log_prob(torch.ones((1, 2))).mean()
loss.backward()
print(mvn.loc.grad)

I get None. I tried fiddling with the dimensions of the both the loc and the scale_tril parameters. Nothing appears to work. Any ideas?

I can obviously implement this myself, but I have a strong preference using existing tools.

Bests,
Eyal.

Output:
You are not calling .grad on your leaf nodes (on .view rather than tensor itself), also you have requires_grad=False on a mean, lets make things more explicit
import torch
from torch.distributions import MultivariateNormal

mean = torch.tensor([0.0, 0.0], requires_grad=True)
cov = torch.tensor([[1.0 , 0.0], [0.0, 1.0]], requires_grad=True)

mvn = MultivariateNormal(loc=mean.view(1,2),
                         scale_tril=cov.view(-1, 2, 2))

loss = -mvn.log_prob(torch.ones((1, 2))).mean()
loss.backward()

print(mean.grad)
print(cov.grad)


Instruction: How to correctly define weights and biases in a PyTorch neural network to get output of correct shape?

Input:
I'm attempting to pass some data of shape [70,1], [70,1,1], and [70,1] into a neural network of linear layers for which I have assigned weights and biases. I am expecting an output of shape [70,1], but I am constantly getting the following error:
RuntimeError                              Traceback (most recent call last)
Input In [78], in <cell line: 1>()
----> 1 output = net(s, h, ep)

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

Input In [68], in NeuralNetHardeningModel.forward(self, s, h, ep)
    101     y = torch.stack((s_eval, h_eval, ep_eval), 1)
    103     print(y.shape, 'y')
--> 105     y1 = self.nn(y)
    107     print(y1.shape, 'y1')
    109 return y1

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/nn/modules/container.py:139, in Sequential.forward(self, input)
    137 def forward(self, input):
    138     for module in self:
--> 139         input = module(input)
    140     return input

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/nn/modules/module.py:1130, in Module._call_impl(self, *input, **kwargs)
   1126 # If we don't have any hooks, we want to skip the rest of the logic in
   1127 # this function, and just call forward.
   1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1129         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130     return forward_call(*input, **kwargs)
   1131 # Do not call functions when jit is used
   1132 full_backward_hooks, non_full_backward_hooks = [], []

File ~/opt/anaconda3/lib/python3.9/site-packages/torch/nn/modules/linear.py:114, in Linear.forward(self, input)
    113 def forward(self, input: Tensor) -> Tensor:
--> 114     return F.linear(input, self.weight, self.bias)

RuntimeError: mat2 must be a matrix, got 1-D tensor

After some inspection, I could not figure out how to solve the error, but I suspect it might have something to do with the way I assign the weights to each layer in the neural net.
Here is the code that I use to define my PyTorch neural network:
# define the NN
def __init__(self, weight, bias, weight_last, bias_last):

    # weight.shape = [3,3,3]
    # bias.shape = [3,3]
    # weight_last = [3], last layer
    # bias_last = [1], last layer

    super(NeuralNetHardeningModel, self).__init__()

    self.weight = weight
    self.bias = bias

    self.weight_last = weight_last
    self.bias_last = bias_last

    self.nn = nn.Sequential(

       nn.Linear(3, 3),
       nn.ReLU(),

       nn.Linear(3, 3),
       nn.ReLU(),

       nn.Linear(3, 3),
       nn.ReLU(),

       nn.Linear(3, 1)
            )

    if len(weight.shape) == 3:

       with torch.no_grad():

           self.nn[0].weight = nn.Parameter(weight[0])
           self.nn[0].bias = nn.Parameter(bias[0])

           self.nn[2].weight = nn.Parameter(weight[1])
           self.nn[2].bias = nn.Parameter(bias[1])

           self.nn[4].weight = nn.Parameter(weight[2])
           self.nn[4].bias = nn.Parameter(bias[2])

           self.nn[6].weight = nn.Parameter(weight_last)
           self.nn[6].bias = nn.Parameter(bias_last)

Here is the code that I use to define my forward pass for my PyTorch neural network:
# forward method for the NN
def forward(self, a, b, c):

  for i in range(a.shape[0]):

     a_eval = torch.flatten(a)  # [70,1]
     b_eval = torch.flatten(b)  # [70,1,1]
     c_eval = torch.flatten(c) # [70,1]

     y = torch.stack((a_eval, b_eval, c_eval), 1)

     y1 = self.nn(y)

  return y1

Output: The shape of weight_last must be [1,3] instead of just [3] to prevent the matrix multiplication error.


Instruction: Generating text/csv file for image path and mask path for semantic segmentation

Input:
I have a huge set of images(60k) and masks(60k) that need to be loaded into a PyTorch dataloader for semantic segmentation.
Directory Structure:

 - Segmentation
       -images
           -color_left_trajectory_3000_00001.jpg
           -color_left_trajectory_3000_00002.jpg
           ...
       -masks
           -color_segmentation_3000_00001.jpg
           -color_segmentation_3000_00002.jpg
           ...

I want to know the most efficient way to load these into a dataloader in Pytorch. I was thinking of generating a csv file with the paths to images and masks. How will I go about generating the same? Any other suggestions are appreciated!

Output:
I recommend that you make a custom subclass from the dataset class. In the init function, the paths to the images and masks are generated and then saved.
This is an example:
import torch
from torch.utils.data import Dataset, DataLoader
import os
from PIL import Image

class CustomData(Dataset):
    def __init__(self,data_dir='Segmentation', data_transform=None,split= 'train'):
        self.imgs = []
        self.labels= []
        self.transform = data_transform
        self.data_dir = data_dir
        #self.imgs_dir = os.path.join(data_dir, split, 'images')
        #self.labels_dir = os.path.join(data_dir, split, 'labels')
        self.imgs_dir = os.path.join(data_dir, 'images')
        self.labels_dir = os.path.join(data_dir, 'labels')
        for img_name in os.listdir(self.imgs_dir):
            img_path = os.path.join(self.imgs_dir, img_name)
            label_name = "color_segmentation_"+"_".join(img.split('.')[0].split('_')[-2:])+'.png'
            label_path = os.path.join(self.labels_dir, label_name)
            self.imgs.append(img_path)
            self.labels.append(label_path)

    def __len__(self):
        return len(self.imgs)

    def __getitem__(self, idx):
        img = Image.open(self.imgs[idx])
        label =  Image.open(self.labels[idx])
        if self.transform is not None:
            img, label = self.transform(img, label)
        return img, label

class ToTensor:
    def __call__(self, image, target=None):
        image = F.to_tensor(image)
        if target is not None:
            target = torch.as_tensor(np.array(target), dtype=torch.int64)
        return image, target
if __name__ == '__main__':
    data = CustomData(data_transform=ToTensor)
    dataloader = DataLoader(data,batch_size=10)


Instruction: python - TypeError: __init__() got an unexpected keyword argument 'checkpoint_callback'

Input:
I'm getting this error message:
TypeError                                 Traceback (most recent call last)
<ipython-input-41-2892cdd4e738> in <module>()
      5   max_epochs=N_EPOCHS,
      6   gpus=1, #GPU
----> 7   progress_bar_refresh_rate=30
      8 )

/usr/local/lib/python3.7/dist-packages/pytorch_lightning/utilities/argparse.py in insert_env_defaults(self, *args, **kwargs)
    343
    344         # all args were already moved to kwargs
--> 345         return fn(self, **kwargs)
    346
    347     return cast(_T, insert_env_defaults)

TypeError: __init__() got an unexpected keyword argument 'checkpoint_callback'

... when I run this chunk:
trainer = pl.Trainer(
  logger=logger,
  checkpoint_callback=checkpoint_callback,
  callbacks=[early_stopping_callback],
  max_epochs=N_EPOCHS,
  gpus=1, #GPU
  progress_bar_refresh_rate=30
)

The 'checkpoint_callback' is defined like this:
checkpoint_callback = ModelCheckpoint(
  dirpath="checkpoints",
  filename="best-checkpoint",
  save_top_k=1,
  verbose=True,
  monitor="val_loss",
  mode="min"
)

I can't figure out what's causing the error - can anyone help me?
View full source code here: https://colab.research.google.com/drive/1hT7PDVb0oGSpLejMGFBMWzRKTPwsSwwS?usp=sharing

Output:
As I am looking into pytorch_lightning github, I do not see checkpoint_callback variable in init (https://github.com/Lightning-AI/lightning/blob/master/src/pytorch_lightning/trainer/trainer.py)
Are you sure thats how it's called? What do you want to achieve by passing this checkpoint_callback?
//edit:
I think you just have to append checkpoint_callback to callbacks list


Instruction: Add new elements to Pytorch tensor

Input:
I have a tensor that looks like this:

I would like to add 0 to each inner tensor. So we have something like[0.1111, 0.6667, 0].
Any suggestion would be appreciated.

Output:
You can do :
z = torch.zeros((10,1))
torch.cat((your_tensor,z),1)