Instruction: RuntimeError: Given groups=1, weight of size [64, 1, 4, 4], expected input[256, 3, 32, 32] to have 1 channels, but got 3 channels instead

Input:
Could you help me fix the above error? If I were to load the mnist dataset, there is no error popping up. The error has to do with the dimension of the other datasets, cifar10, fmnist and so on and cannot be run when applied to these sets. Any help appreciated.

# noinspection PyUnresolvedReferences
import os
# imports
# noinspection PyUnresolvedReferences
import pickle
from time import time

from torchvision import datasets, transforms
from torchvision.utils import save_image

import site
site.addsitedir('/content/gw_gan/model')
from loss import gwnorm_distance, loss_total_variation, loss_procrustes
from model_cnn import Generator, Adversary
from model_cnn import weights_init_generator, weights_init_adversary

# internal imports
from utils import *

# get arguments
args = get_args()

# system preferences
seed = np.random.randint(100)
torch.set_default_dtype(torch.double)
np.random.seed(seed)
torch.manual_seed(seed)

# settings
batch_size = 256
z_dim = 100
lr = 0.0002
ngen = 3
beta = args.beta
lam = 0.5
niter = 10
epsilon = 0.005
num_epochs = args.num_epochs
cuda = args.cuda
channels = args.n_channels
id1 = args.id

model = 'gwgan_{}_eps_{}_tv_{}_procrustes_{}_ngen_{}_channels_{}_{}' \
        .format(args.data, epsilon, lam, beta, ngen, channels, id1)
save_fig_path = 'out_' + model
if not os.path.exists(save_fig_path):
    os.makedirs(save_fig_path)

# data import
dataloader = torch.utils.data.DataLoader(
        datasets.CIFAR10('./data/cifar10', train=True, download=True,
                         transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.5, 0.5, 0.5),
                                                 (0.5, 0.5, 0.5))])),
        batch_size=batch_size, drop_last=True, shuffle=True)

# print example images
save_image(next(iter(dataloader))[0][:25],
           os.path.join(save_fig_path, 'real.pdf'), nrow=5, normalize=True)

# define networks and parameters
generator = Generator(output_dim=channels)
adversary = Adversary(input_dim=channels)

# weight initialisation
generator.apply(weights_init_generator)
adversary.apply(weights_init_adversary)

if cuda:
    generator = generator.cuda()
    adversary = adversary.cuda()

# create optimizer
g_optimizer = torch.optim.Adam(generator.parameters(), lr, betas=(0.5, 0.99))
# zero gradients
generator.zero_grad()

c_optimizer = torch.optim.Adam(adversary.parameters(), lr, betas=(0.5, 0.99))
# zero gradients
adversary.zero_grad()

# sample for plotting
num_test_samples = batch_size
z_ex = torch.randn(num_test_samples, z_dim)
if cuda:
    z_ex = z_ex.cuda()

loss_history = list()
loss_tv = list()
loss_orth = list()
loss_og = 0
is_hist = list()

for epoch in range(num_epochs):
    t0 = time()

    for it, (image, _) in enumerate(dataloader):
        train_c = ((it + 1) % (ngen + 1) == 0)

        x = image.double()
        if cuda:
            x = x.cuda()

        # sample random number z from Z
        z = torch.randn(image.shape[0], z_dim)

        if cuda:
            z = z.cuda()

        if train_c:
            for q in generator.parameters():
                q.requires_grad = False
            for p in adversary.parameters():
                p.requires_grad = True
        else:
            for q in generator.parameters():
                q.requires_grad = True
            for p in adversary.parameters():
                p.requires_grad = False

        # result generator
        g = generator.forward(z)

        # result adversary
        f_x = adversary.forward(x)
        f_g = adversary.forward(g)

        # compute inner distances
        D_g = get_inner_distances(f_g, metric='euclidean', concat=False)
        D_x = get_inner_distances(f_x, metric='euclidean', concat=False)

        # distance matrix normalisation
        D_x_norm = normalise_matrices(D_x)
        D_g_norm = normalise_matrices(D_g)

        # compute normalized gromov-wasserstein distance
        loss, T = gwnorm_distance((D_x, D_x_norm), (D_g, D_g_norm),
                                  epsilon, niter, loss_fun='square_loss',
                                  coupling=True, cuda=cuda)

        if train_c:
            # train adversary
            loss_og = loss_procrustes(f_x, x.view(x.shape[0], -1), cuda)
            loss_to = -loss + beta * loss_og
            loss_to.backward()

            # parameter updates
            c_optimizer.step()
            # zero gradients
            reset_grad(generator, adversary)

        else:
            # train generator
            loss_t = loss_total_variation(g)
            loss_to = loss + lam * loss_t
            loss_to.backward()

            # parameter updates
            g_optimizer.step()
            # zero gradients
            reset_grad(generator, adversary)

    # plotting
    # get generator example
    g_ex = generator.forward(z_ex)
    g_plot = g_ex.cpu().detach()

    # plot result
    save_image(g_plot.data[:25],
               os.path.join(save_fig_path, 'g_%d.pdf' % epoch),
               nrow=5, normalize=True)

    fig1, ax = plt.subplots(1, 3, figsize=(15, 5))
    ax0 = ax[0].imshow(T.cpu().detach().numpy(), cmap='RdBu_r')
    colorbar(ax0)
    ax1 = ax[1].imshow(D_x.cpu().detach().numpy(), cmap='Blues')
    colorbar(ax1)
    ax2 = ax[2].imshow(D_g.cpu().detach().numpy(), cmap='Blues')
    colorbar(ax2)
    ax[0].set_title(r'$T$')
    ax[1].set_title(r'inner distances of $D$')
    ax[2].set_title(r'inner distances of $G$')
    plt.tight_layout(h_pad=1)
    fig1.savefig(os.path.join(save_fig_path, '{}_ccc.pdf'.format(
            str(epoch).zfill(3))), bbox_inches='tight')

    loss_history.append(loss)
    loss_tv.append(loss_t)
    loss_orth.append(loss_og)
    plt.close('all')

# plot loss history
fig2 = plt.figure(figsize=(2.4, 2))
ax2 = fig2.add_subplot(111)
ax2.plot(loss_history, 'k.')
ax2.set_xlabel('Iterations')
ax2.set_ylabel(r'$\overline{GW}_\epsilon$ Loss')
plt.tight_layout()
plt.grid()
fig2.savefig(save_fig_path + '/loss_history.pdf')

fig3 = plt.figure(figsize=(2.4, 2))
ax3 = fig3.add_subplot(111)
ax3.plot(loss_tv, 'k.')
ax3.set_xlabel('Iterations')
ax3.set_ylabel(r'Total Variation Loss')
plt.tight_layout()
plt.grid()
fig3.savefig(save_fig_path + '/loss_tv.pdf')

fig4 = plt.figure(figsize=(2.4, 2))
ax4 = fig4.add_subplot(111)
ax4.plot(loss_orth, 'k.')
ax4.set_xlabel('Iterations')
ax4.set_ylabel(r'$R_\beta(f_\omega(X), X)$ Loss')
plt.tight_layout()
plt.grid()
fig4.savefig(save_fig_path + '/loss_orth.pdf')

The error displays:
Traceback (most recent call last):
  File "/content/gw_gan/main_gwgan_cnn.py", line 160, in <module>
    f_x = adversary.forward(x)
  File "/content/gw_gan/model/model_cnn.py", line 62, in forward
    x = self.conv(input)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py", line 117, in forward
    input = module(input)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 423, in forward
    return self._conv_forward(input, self.weight)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 420, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: Given groups=1, weight of size [64, 1, 4, 4], expected input[256, 3, 32, 32] to have 1 channels, but got 3 channels instead

This is for an application of a generative model, where this is a CNN. The reference this is taken from is from main_gwgan_cnn at https://github.com/bunnech/gw_gan. A GAN is proposed to learn from incomparable spaces and produce results.

Output:
You have to set the --n_channels otherwise args.n_chanels will default to 1 as see here. The example given here is for FMNIST which has a single channel.
You are running on CIFAR, you should set it to 3 since there are three channels.


Instruction: pytorch MNIST neural network produces several non-zero outputs

Input:
I tried to do a neural network that operates on MNIST data set. I was mostly following the pytorch.nn tutorial. As a result, i got a model that learns, but there's something wrong with the process or with the model itself. Instead of one active neuron at the output, i recieve multiple ones.
Here's the model itself:
model = nn.Sequential(
    nn.Linear(784, 64),
    nn.ReLU(),
    nn.Linear(64, 10),
    nn.ReLU(),
)

And here's the training process:
loss_func = nn.CrossEntropyLoss()
opt = optim.SGD(model.parameters(), lr=lr)

for epoch in range(epochs):
    model.train()
    for xbt, ybt in train_dl:
        pred = model(xbt)
        loss = loss_func(pred, ybt)
        opt.zero_grad()
        loss.backward()
        opt.step()

    model.eval()
    # Validation
    if epoch % 10 == 0:
        with torch.no_grad():
            losses, nums = zip(
                *[(loss_func(model(xbv), ybv), len(xbv)) for xbv, ybv in valid_dl]
            )
        val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)

        print(epoch, val_loss)

Here's average loss each 10th epoch:
0 0.13384412774592638
10 0.0900113809091039
20 0.09795805384699234
30 0.10341344920364791
40 0.10804545368137551

And thats how result of applying the model to the validation set looks like:
[[ 0.         0.         0.        ... 28.436266   0.         5.001435 ]
 [ 7.3331523 12.666427  31.898096  ...  0.         0.         0.       ]
 [ 0.        18.116354   8.049953  ...  4.330721   0.         0.       ]
 ...
 [ 8.504517   0.         6.302228  ...  0.         0.         0.       ]
 [ 1.7339934  0.         0.        ...  0.         2.1565871  0.       ]
 [45.750134   0.         6.2685804 ...  2.247082   0.         0.       ]]
 Shape: (9984, 10)

I tried changing learning speed, model layers, amount of epochs, but nothing seems to work.

Output: You have 10 neurons with ReLU in the last layers and yes all the neurons will fire/activated. In this case every neuron applies a ReLu function on the output of linear activation. ie ReLu(w.x+b). There are 10 such neurons and all of them will give out certain output based on its input, and yes all of them get fired/activated. The way you infer an output from this is by taking the class corresponding to the neuron which has the hugest activation (using np.argmax or torch.max).


Instruction: Problem with forward step in pytorch model

Input:
Traceback saying that:
* Epoch 1/20
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-24-4f0f868c6227> in <module>()
      1 max_epochs = 20
      2 optim = torch.optim.Adam(model.parameters(), lr=1e-3)
----> 3 train(model, optim, bce_loss, max_epochs, data_tr, data_val)

2 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

TypeError: forward() takes 2 positional arguments but 3 were given

But in train section of my net I did not pass more than two arguments(including self argument)
this is link to my code
Maybe problem not in train

Output:
Your model has sub-modules for which you (implicitly) call forward with more than one argument:
x = self.dec_conv3(self.unpool3(x, indices3))

Your unpool are simply MaxPool layers - they do not expect two input arguments.


Instruction: What is the difference between Neural Network Frameworks and RL Algorithm Libraries?

Input:
I know this is a silly question, but I cannot find a good way to put it.
I've worked with TensorFlow and TFAgents, and am now moving to Ray RLlib. Looking at all the RL frameworks/libraries, I got confused about the difference between the two below:

frameworks such as Keras, TensorFlow, PyTorch
RL implementation libraries such as TFAgents, RLlib, OpenAi Baseline, Tensorforce, KerasRL, etc

For example, there are Keras codes in TensorFlow and Ray RLlib supports both TensorFlow and PyTorch. How are they all related?
My understanding so far is that Keras allows to make neural networks and TensorFlow is more of a math library for RL (I don't have enough understanding about PyTorch). And libraries like TFAgents and RLlib use frameworks like Keras and TensorFlow to implement existing RL algorithms so that programmers can utilize them with ease.
Can someone please explain how they are interconnected/different? Thank you very much.

Output:
Yes you are kind of right. Frameworks like Keras, TF (which also uses keras btw) and Pytorch are general Deep Learning frameworks. For most artificial neural network use-cases these frameworks work just fine and your typical pipeline is going to look something like:

Preprocess your dataset
Select an appropriate model for this problem setting
model.fit(dataset)
Analyze results

Reinforcement Learning though is substantially different from most other Data Science ML applications. To start with, in RL you actually generate your own dataset by having your model (the Agent) interact with an environment; this complicates the situation substantially particularly from a computational standpoint. This is because in the traditional ML scenario most of the computational heavy-lifting is done by that model.fit() call. And the good thing about the aforementioned frameworks is that from that call your code actually enters very efficient C/C++ code (usually also implementing CUDA libraries to use the GPU).
In RL the big problem is the environment that the agent interacts with. I separate this problem in two parts:
a) The environment cannot be implemented in these frameworks because it will always change based on what you are doing. As such you have to code the environment and - chances are - it's not gonna be very efficient.
b) The environment is a key component in the code and it constantly intreacts multiple times with your Agent, and there are multiple ways in which that interaction can be mediated.
These two factors lead to the necessity to standardize the environment and the interaction between it and the agent. This standardization allows for highly reusable code and also code that is more interpretable by others in how it exactly operates. Furthermore it is possible this way to, for example, easily run parallel environments (TF-agents allows this for example) even though your environment object is not really written to manage this.
RL frameworks are thus providing this standardization and features that come with it. Their relation to Deep Learning frameworks is that RL libraries often come with a lot of pre-implemented and flexible agent architectures that have been among the most relevant in the literature. These agents are usually nothing more than a some fancy ANN architecture wrapped in some class that standardizes their operation within the given RL framework. Therefore as a backend for these ANN models, RL frameworks use DL frameworks to run the computations efficiently.


Instruction: How to get the image torchvision.utils.save_image saves, without reading it back from disk?

Input:
from torchvision.utils import save_image

...
save_image(im, f'im_name.png')

In my case (standard mnist), using code from here, im is a Tensor:96, and save_image works.
I want that image in memory to show it in other plots, and I don't want to read it back after saving it, which seems kind of stupid.
Is there a way to separate the functionality of generating the image and of saving it?

Edit
clarification:
I want an equivalent to
save_image(im, f'im_name.png')
reread = plt.imread(f'im_name.png')

without saving the image and reading it back.
I just want the image, and I want to save it later.
the save_image function does some work, like stacking multiple images into one, converting the tensor to images of correct sizes and so on. I want only that part without the saving to disk.

Output:
About 2 weeks later, I stumbled upon the solution by accident.
grid = torchvision.utils.make_grid(im)

grid will be the image save_image was just saving.


Instruction: Pytorch: No training effect after deepcopy

Input:
I tried to make a copy of a neural network in pytorch and subsequently train the copied network, but training does not seem to change the weights in the network after copying. This post suggests that deepcopy is a convenient way to make a copy of a neural network, so I tried using that in my code.
The code below works just fine and shows that the weights and accuracy of the network are different after training from before training. However, when I toggle so that network_cp=deepcopy(network) and optimizer_cp=deepcopy(optimizer), the accuracy and weights before and after training are exactly the same.
# torch settings
torch.backends.cudnn.enabled = True
device = torch.device("cpu")

# training settings
learning_rate = 0.01
momentum = 0.5
batch_size_train = 64
batch_size_test = 1000

# get MNIST data set
train_loader, test_loader = load_mnist(batch_size_train=batch_size_train,
    batch_size_test=batch_size_test)

# make a network
network = Net()
optimizer = optim.SGD(network.parameters(), lr=learning_rate,
    momentum=momentum)
network.to(device)

# train network
train(network, optimizer, train_loader, device)

# copy network
network_cp = network
#network_cp = deepcopy(network)
optimizer_cp = optimizer
#optimizer_cp = deepcopy(optimizer)

# get edge weights and accuracy of the copied network
acc1 = float(test(network_cp, optimizer_cp, test_loader, device))
weights1 = np.array(get_edge_weights(network_cp))

# train copied network
train(network_cp, optimizer_cp, train_loader, device)

# get edge weights and accuracy of the copied network after training
acc2 = float(test(network_cp, optimizer_cp, test_loader, device))
weights2 = np.array(get_edge_weights(network_cp))

# compare edge weights and accuracy of copied network before and after training
print('accuracy', acc1, acc2)
print('abs diff of weights for net1 and net2', np.sum(np.abs(weights1-weights2)))

To run the code above, include these imports and function definitions:
import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
import torch.nn as tnn
import torch.nn.functional as tnf
from copy import deepcopy
import numpy as np

def load_mnist(batch_size_train = 64, batch_size_test = 1000):
    train_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST('temp/', #'/data/users/alice/pytorch_training_files/',
                                   train=True, download=True,
                                   transform=torchvision.transforms.Compose([
                                       torchvision.transforms.ToTensor(),
                                       torchvision.transforms.Normalize(
                                           (0.1307,), (0.3081,))
                                       ])),
        batch_size=batch_size_train, shuffle=True)

    test_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST('temp/', #'/data/users/alice/pytorch_training_files/',
                                   train=False, download=True,
                                   transform=torchvision.transforms.Compose([
                                       torchvision.transforms.ToTensor(),
                                       torchvision.transforms.Normalize(
                                           (0.1307,), (0.3081,))
                                     ])),
        batch_size=batch_size_test, shuffle=True)

    return(train_loader, test_loader)

def train(network, optimizer, train_loader, device, n_epochs=5):
    network.train()
    for epoch in range(1, n_epochs + 1):
        for batch_idx, (data, target) in enumerate(train_loader):
            data, target = data.to(device), target.to(device)
            optimizer.zero_grad()
            output = network(data)
            loss = tnf.nll_loss(output, target)
            loss.backward()
            optimizer.step()

def test(network, optimizer, test_loader, device):
    network.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = network(data)
            test_loss += tnf.nll_loss(output, target, size_average=False).item()
            pred = output.data.max(1, keepdim=True)[1]
            correct += pred.eq(target.data.view_as(pred)).sum()
    test_loss /= len(test_loader.dataset)
    print('\nTest set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

    return(float(correct)/float(len(test_loader.dataset)))

def get_edge_weights(network):
    layers = [module for module in network.modules()][1:]
    output = np.zeros(1)
    for j, layer in enumerate(layers):
        weights = list(layer.parameters())[0]
        weights_arr = weights.detach().numpy()
        weights_arr = weights_arr.flatten()
        output = np.concatenate((output,weights_arr))
    return output[1:]

class Net(tnn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 =tnn.Linear(784,264)
        self.fc2 = tnn.Linear(264,10)

    def forward(self, x):
        x = tnf.relu(self.fc1(x.view(-1,784)))
        x = tnf.relu(self.fc2(x))
        return tnf.log_softmax(x)

Output:
After optimizer_cp = deepcopy(optimizer), the optimizer_cp still wants to optimize the old model's parameters (as defined by optimizer = optim.SGD(network.parameters(), lr=learning_rate, momentum=momentum)).
After deep copying the model, the optimizer needs to be told to optimize this new model's parameters:
optimizer_cp = optim.SGD(network_cp.parameters(), lr=learning_rate, momentum=momentum)


Instruction: Pytorch does not pad the layers

Input:
While making a network I used deconvolution layer :
def deconv3d(cin,cout,k=4,s=2,pad=-1):
    pad = (k - 1) // 2 if pad < 0 else pad
    return nn.Sequential(
        nn.ConvTranspose3d(cin,cout,kernel_size=k,stride=s,padding = pad,bias=False),
        nn.ReLU(inplace=True)
    )

And I adjusted this to networks and it keeps reducing the size of layer.
self.conv_f5 = conv3d(128, 128, k=3, s=1, pad=1)
    self.conv_f6 = deconv3d(128,64,k=3,s=2,pad=1)
    self.conv_f7 = conv3d(64,64,k=3,s=1,pad=1)
    self.conv_f8 = deconv3d(64,32,k=3,s=2,pad=1)
    self.conv_f9 = conv3d(32,32,k=3,s=1,pad=1)

This is the layers I made and the result is :
Cost Volume: torch.Size([2, 128, 100, 120, 160])
fc5 torch.Size([2, 128, 50, 60, 80])
fc6 torch.Size([2, 64, 99, 119, 159])
fc8 torch.Size([2, 32, 197, 237, 317])

I cannot understand why f3 has smaller size than expected and finding how to fix it.
Please tell me how and thank you very much.

Output:
This is still not a reproducible example. I see there are calls to conv3d for which you have not provided the source code. I believe it is not a mere wrapper around nn.Conv3d because the resulting size does not match (self.conv_5for example should not change the tensor shape with its given parameters)
However I believe what puzzles you is the odd size ([2, 64, 99, 119, 159]). In that case you want to read very carefully the documentation for nn.ConvTranspose3d. In particular :
However, when stride > 1, Conv3d maps multiple input shapes to the same output shape. output_padding is provided to resolve this ambiguity by effectively increasing the calculated output shape on one side. Note that output_padding is only used to find output shape, but does not actually add zero-padding to output.
According to the shape equation which is provided with the documentation, I believe you want to add an output_padding=1 to your deconv layers.


Instruction: Cannot install torchvision via poetry in windows

Input:
I succeeded in pytorch installation thanks to answers here
Poetry and PyTorch.
But, I'm still failed to install torchvision via poetry.
> poetry add torchvision==0.8.2

Updating dependencies
Resolving dependencies...

Writing lock file

Package operations: 1 install, 0 updates, 0 removals

  • Installing torchvision (0.8.2)

  RuntimeError

  Unable to find installation candidates for torchvision (0.8.2)

  at ~\.poetry\lib\poetry\installation\chooser.py:72 in choose_for
       68│
       69│             links.append(link)
       70│
       71│         if not links:
    →  72│             raise RuntimeError(
       73│                 "Unable to find installation candidates for {}".format(package)
       74│             )
       75│
       76│         # Get the best link

Failed to add packages, reverting the pyproject.toml file to its original content.

I googled it and found some answers that say 'just pip install torchvision'.
But I'm suspicious that it works because according to PyPi(https://pypi.org/project/torchvision/#files), there is no wheel file for windows. And I tried it and it failed as I expected.
Is there any way to install latest torchvisin which is compatible with latest torch(1.7.1) in windows? + via poetry?

Output:
Look in https://download.pytorch.org/whl/torch_stable.html for the version you want to install (torchvision version, python version, CUDA version, OS etc.)
Add a URL dependency to your pyproject.toml file. For example I have torchvision 1.8.0 working with the following in my dependencies

[tool.poetry.dependencies]
python = "^3.8"
torch = {url = "https://download.pytorch.org/whl/cu102/torch-1.8.0-cp38-cp38-win_amd64.whl"}
torchvision = {url = "https://download.pytorch.org/whl/cu102/torchvision-0.9.0-cp38-cp38-win_amd64.whl"}

From your activated virtual environment do poetry update torchvision and you should be good to go.


Instruction: How to understand creating leaf tensors in PyTorch?

Input:
From PyTorch documentation:
b = torch.rand(10, requires_grad=True).cuda()
b.is_leaf
False
# b was created by the operation that cast a cpu Tensor into a cuda Tensor

e = torch.rand(10).cuda().requires_grad_()
e.is_leaf
True
# e requires gradients and has no operations creating it

f = torch.rand(10, requires_grad=True, device="cuda")
f.is_leaf
True
# f requires grad, has no operation creating it

But why are e and f leaf Tensors, when they both were also cast from a CPU Tensor, into a Cuda Tensor (an operation)?
Is it because Tensor e was cast into Cuda before the in-place operation requires_grad_()?
And because f was cast by assignment device="cuda" rather than by method .cuda()?

Output:
When a tensor is first created, it becomes a leaf node.
Basically, all inputs and weights of a neural network are leaf nodes of the computational graph.
When any operation is performed on a tensor, it is not a leaf node anymore.
b = torch.rand(10, requires_grad=True) # create a leaf node
b.is_leaf # True
b = b.cuda() # perform a casting operation
b.is_leaf # False

requires_grad_() is not an operation in the same way as cuda() or others are.
It creates a new tensor, because tensor which requires gradient (trainable weight) cannot depend on anything else.
e = torch.rand(10) # create a leaf node
e.is_leaf # True
e = e.cuda() # perform a casting operation
e.is_leaf # False
e = e.requires_grad_() # this creates a NEW tensor
e.is_leaf # True

Also, detach() operation creates a new tensor which does not require gradient:
b = torch.rand(10, requires_grad=True)
b.is_leaf # True
b = b.detach()
b.is_leaf # True

In the last example we create a new tensor which is already on a cuda device.
We do not need any operation to cast it.
f = torch.rand(10, requires_grad=True, device="cuda") # create a leaf node on cuda


Instruction: gaussianblur transform not found in torchvision.transforms

Input:
I have written the following data augmentation pipeline for Pytorch:
transform = transforms.Compose([
            transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.GaussianBlur(11, sigma=(0.1, 2.0)),
            transforms.ToTensor(),
            transforms.Normalize(mean, std)
        ])

But running the above code gives me the following error:
AttributeError: module 'torchvision.transforms' has no attribute 'GaussianBlur'

Is GaussianBlur a new feature that has not been included in torchvision yet? Or is it just my torchvision version that is too old?
I found it in the following documentation page: torchvision.transforms
Here are my packages versions:
torch                    1.6.0+cu101
torchvision              0.7.0+cu101

Output:
On this documentation page, you can look for features on the different versions of pytorch (change version in the upper left corner). It appears that GaussianBlur did not exist in pytorch 1.6, and was added in 1.7. Your code should work with the latest version.
If interested, here is the relevant merge commit : https://github.com/pytorch/vision/pull/2658


Instruction: Clarification of a Faster R-CNN torchvision implementation

Input:
I'm digging through the source code of the Faster R-CNN implementation of torchvision and I'm facing some things I don't quite understand. Namely, assuming that I want to create a Faster R-CNN model, not pretrained on COCO, with a backbone pre-trained on ImageNet, and then just get the backbone I do the following:
plain_backbone = fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=True).backbone.body

Which is consistent with how the backbone is set-up as indicated here and here. However, when I pass an image through the model, the results don't correspond to what I would obtain if I just set-up a resnet50 directly. Namely:
# Regular resnet50, pretrained on ImageNet, without the classifier and the average pooling layer
resnet50_1 = torch.nn.Sequential(*(list(torchvision.models.resnet50(pretrained=True).children())[:-2]))
resnet50_1.eval()
# Resnet50, extract from the Faster R-CNN, also pre-trained on ImageNet
resnet50_2 = fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=True).backbone.body
resnet50_2.eval()
# Loading a random image, converted to torch.Tensor, rescalled to [0, 1] (not that it matters)
image = transforms.ToTensor()(Image.open("random_images/random.jpg")).unsqueeze(0)
# Obtaining the model outputs
with torch.no_grad():
    # Output from the regular resnet50
    output_1 = resnet50_1(image)
    # Output from the resnet50 extracted from the Faster R-CNN
    output_2 = resnet50_2(image)["3"]
    # Their outputs aren't the same, which I would assume they should be
    np.testing.assert_almost_equal(output_1.numpy(), output_2.numpy())

Looking forward to your thoughts!

Output:
This is because fasterrcnn_resnet50_fpn uses a custom normalization layer (FrozenBatchNorm2d) instead of the default BatchNorm2D. They are very similar but I suspect that the small numerical differences are causing issues.
It will pass the check if you specify the same normalization layer to be used for the standard resnet:
import torch
import torchvision
from torchvision.models.detection.faster_rcnn import fasterrcnn_resnet50_fpn
import numpy as np
from torchvision.ops import misc as misc_nn_ops

# Regular resnet50, pretrained on ImageNet, without the classifier and the average pooling layer
resnet50_1 = torch.nn.Sequential(*(list(torchvision.models.resnet50(pretrained=True, norm_layer=misc_nn_ops.FrozenBatchNorm2d).children())[:-2]))
resnet50_1.eval()
# Resnet50, extract from the Faster R-CNN, also pre-trained on ImageNet
resnet50_2 = fasterrcnn_resnet50_fpn(pretrained=False, pretrained_backbone=True).backbone.body
resnet50_2.eval()
# am too lazy to get a real image
image = torch.ones((1, 3, 224, 224))
# Obtaining the model outputs
with torch.no_grad():
    # Output from the regular resnet50
    output_1 = resnet50_1(image)
    # Output from the resnet50 extracted from the Faster R-CNN
    output_2 = resnet50_2(image)["3"]
    # Passes
    np.testing.assert_almost_equal(output_1.numpy(), output_2.numpy())


Instruction: Why is the decoder in an autoencoder uses a sigmoid on the last layer?

Input:
I am looking at this working variational auto encoder.
The main class
class VAE(nn.Module):
    def __init__(self):
        super(VAE, self).__init__()

        self.fc1 = nn.Linear(784, 400)
        self.fc21 = nn.Linear(400, 20)
        self.fc22 = nn.Linear(400, 20)
        self.fc3 = nn.Linear(20, 400)
        self.fc4 = nn.Linear(400, 784)

    def encode(self, x):
        h1 = F.relu(self.fc1(x))
        return self.fc21(h1), self.fc22(h1)

    def reparametrize(self, mu, logvar):
        std = logvar.mul(0.5).exp_()
        if torch.cuda.is_available():
            eps = torch.cuda.FloatTensor(std.size()).normal_()
        else:
            eps = torch.FloatTensor(std.size()).normal_()
        eps = Variable(eps)
        return eps.mul(std).add_(mu)

    def decode(self, z):
        h3 = F.relu(self.fc3(z))
        return F.sigmoid(self.fc4(h3))

    def forward(self, x):
        mu, logvar = self.encode(x)
        z = self.reparametrize(mu, logvar)
        return self.decode(z), mu, logvar

has
    def decode(self, z):
        h3 = F.relu(self.fc3(z))
        return F.sigmoid(self.fc4(h3))

I can't explain to myself why the last layer should be passed through a sigmoid before returning.
Please explain.

EDIT:
I just checked without the sigmoid. Results are still nice.
Now I am not sure if it is needed or not.

Output:
As mentioned in the answer by Jim J, sigmoid forces the output to the range [0, 1]. In this case, it's not because we want to interpret the output as a probability, rather it's done to force the output to be interpreted as pixel intensity of a grey scale image.
If you remove the sigmoid, the NN will have to learn that all the outputs should be in the range [0, 1]. The sigmoid might help making the learning process more stable.


Instruction: Getting nan as loss value

Input:
I have implemented focal loss in Pytorch with using of this paper. And ran into a problem with loss - got nan as loss function value.
This is implementation of focal loss:
def focal_loss(y_real, y_pred, gamma = 2):
    y_pred = torch.sigmoid(y_pred)
    return -torch.sum((1 - y_pred)**gamma * y_real * torch.log(y_pred) +
                       y_pred**gamma * (1 - y_real) * torch.log(1 - y_pred))

Train loop and my SegNet are working, I think so, because I have tested them with dice and bce losses.
I think errors occurs in backprop. Why can it be? Maybe my implementation is wrong?

Output:
This version is working:
def focal_loss(y_real, y_pred, eps = 1e-8, gamma = 0):
    probabilities = torch.clamp(torch.sigmoid(y_pred), min=eps, max=1-eps)
    return torch.mean((1 - probabilities)**gamma *
           (y_pred - y_real * y_pred + torch.log(1 + torch.exp(-y_pred))))


Instruction: GPyTorch, how to set initial value for "lengthscale" hyperparameter?

Input:
I am using GPyTorch regressor according to the documentation.
I would like to set an initial value for the "lengthscale" hyperparameter in RBF kernel.
I want to set a constant number as initial value for "lengthscale" (similar to what we can do in scikit-learn Gaussian Process Regressor).
If you have any idea, please let me know.

Output:
There are two cases that follow from your question:

You want to initialize your lengthscale with some value but the lengthscale is then optimized on further by the optimizer
Assuming you have the same model as given in the documentation you have linked, just add the following before your training loop:
init_lengthscale = 0.1
model.covar_module.base_kernel.lengthscale = init_lengthscale

The model.covar_module gets your entire kernel and the base_kernel gets you your
RBF kernel.

You want to fix your lengthscale as some constant value which will not be further optimized
In addition to the code for the first case, you do not feed the lengthscale as a hyperparameter to be optimized to your optimizer.
all_params = set(exactModel.parameters())
final_params = list(all_params - {exactModel.covar_module.base_kernel.raw_lengthscale})
optimizer = torch.optim.Adam(final_params, lr=0.1)

We remove the set of raw lengthscale values from all_params to create final_params which we then pass to the optimizer.

Some sources to help:

https://github.com/cornellius-gp/gpytorch/issues/689
https://docs.gpytorch.ai/en/v1.1.1/examples/00_Basic_Usage/Hyperparameters.html


Instruction: can I train(optimize) on f1 score loss with pytorch

Input:
I am building a binary classifier like below. Can I replace the BCELoss to optimize f1 score?
criterion = nn.BCELoss()

preds = model(inputs)
loss = criterion(preds , labels)

Output:
F1 score is not a smooth function, so it cannot be optimized directly with gradient descent. With gradually changing network parameters, the output probability changes smoothly but the F1 score only changes when the probability crosses the boundary of 0.5. As a result, the gradient of F1 score is zero almost everywhere.
You can use a soft version of the F-measure as described here. The trick is that you basically replace the count of true positives and false positives with a sort of probabilistic version:

where oi is the network output and ti is the ground truth target probability. Then you continue with computing F-measure as usual.
Also, you might find this Kaggle tutorial useful.


Instruction: no member named 'KInt16' in namespace 'torch'

Input:
I am working on a cpp extension for pytorch, and followed the official tutorial, using libtorch and cmake to compile the program. But I met the problem of creating tensor.
This code can work.
#include <torch/torch.h>
#include <iostream>
int main(){
    std::vector<int64_t> test_data = {1, 2, 3, 4, 5, 6, 7, 8, 9};
    torch::Tensor s = torch::from_blob(test_data.data(), {3, 3}, torch::kInt64);
    std::cout << "test case pass" << std::endl;
}

But this code can't work.
int main(){
    auto option = torch::TensorOptions().dtype(torch::KInt16);
    auto b = torch::zeros({2,3}, option);
    std::cout << "test case pass" << std::endl;
}

and the compile error log is here.
error: no member named 'KInt16' in namespace 'torch'; did you mean 'kInt16'?

Output:
As explicitly stated by the error message, you made a typo : you should have written
torch::kInt16 instead of torch::KInt16. The k should not be capitalized.


Instruction: Pytorch LSTM in ONNX.js - Uncaught (in promise) Error: unrecognized input '' for node: LSTM_4

Input:
I am trying to run a Pytorch LSTM network in browser. But I am getting this error:
graph.ts:313 Uncaught (in promise) Error: unrecognized input '' for node: LSTM_4
    at t.buildGraph (graph.ts:313)
    at new t (graph.ts:139)
    at Object.from (graph.ts:77)
    at t.load (model.ts:25)
    at session.ts:85
    at t.event (instrument.ts:294)
    at e.initialize (session.ts:81)
    at e.<anonymous> (session.ts:63)
    at onnx.min.js:14
    at Object.next (onnx.min.js:14)

How can I resolve this? Here is my code for saving the model to onnx:
net = torch.load('trained_model/trained_model.pt')
net.eval()

with torch.no_grad():
    input = torch.tensor([[1,2,3,4,5,6,7,8,9]])
    h0, c0 = net.init_hidden(1)
    output, (hn, cn) = net.forward(input, (h0,c0))

    torch.onnx.export(net, (input, (h0, c0)), 'trained_model/trained_model.onnx',
                    input_names=['input', 'h0', 'c0'],
                    output_names=['output', 'hn', 'cn'],
                    dynamic_axes={'input': {0: 'sequence'}})

I put input as the only dynamic axis since it is the only one that can vary in size. With this code, the model saves properly as trained_model.onnx. It does give me a warning:
UserWarning: Exporting a model to ONNX with a batch_size other than 1, with a variable length with LSTM can cause an error when running the ONNX model with a different batch size. Make sure to save the model with a batch size of 1, or define the initial states (h0/c0) as inputs of the model.
  warnings.warn("Exporting a model to ONNX with a batch_size other than 1, "

This warning a little confusing since I am exporting it with a batch_size of 1:

input has shape torch.Size([1, 9])
h0 has shape torch.Size([2, 1, 256]) - corresponding to  (num_lstm_layers, batch_size, hidden_dim)
c0 also has shape torch.Size([2, 1, 256])

But since I do define h0/c0 as inputs of the model I don't think this relates to the problem.
This is my javascript code for running in the browser:
<script src="https://cdn.jsdelivr.net/npm/onnxjs/dist/onnx.min.js"></script>
<!-- Code that consume ONNX.js -->
<script>
  // create a session
  const myOnnxSession = new onnx.InferenceSession();

  console.log('trying to load the model')

  // load the ONNX model file
  myOnnxSession.loadModel("./trained_model.onnx").then(() => {

    console.log('successfully loaded model!')

    // after this I generate input and run the model
    // since my code fails before this it isn't relevant
  });
</script>

Based on the console.log statements, it is failing to load to the model. How should I resolve this? If relevant I'm using Python 3.8.5, Pytorch 1.6.0, ONNX 1.8.0.

Output:
For anyone coming across this in the future, I believe I'm getting this error because even though ONNX supports Pytorch LSTM networks, ONNX.js does not support it yet.
To get around this, instead of running in the browser I may use a simple web application framework called streamlit.


Instruction: load pytorch dataloader into GPU

Input:
Is there a way to load a pytorch DataLoader (torch.utils.data.Dataloader) entirely into my GPU?
Now, I load every batch separately into my GPU.
CTX = torch.device('cuda')

train_loader = torch.utils.data.DataLoader(
    train_dataset,
    batch_size=BATCH_SIZE,
    shuffle=True,
    num_workers=0,
)

net = Net().to(CTX)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=LEARNING_RATE)

for epoch in range(EPOCHS):
    for inputs, labels in test_loader:
        inputs = inputs.to(CTX)        # this is where the data is loaded into GPU
        labels = labels.to(CTX)

        optimizer.zero_grad()

        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

    print(f'training accuracy: {net.validate(train_loader, device=CTX)}/{len(train_dataset)}')
    print(f'validation accuracy: {net.validate(test_loader, device=CTX)}/{len(test_dataset)}')

where the Net.validate() function is given by
def validate(self, val_loader, device=torch.device('cpu')):
    correct = 0
    for inputs, labels in val_loader:
        inputs = inputs.to(device)
        labels = labels.to(device)
        outputs = torch.argmax(self(inputs), dim=1)
        correct += int(torch.sum(outputs==labels))
    return correct

I would like to improve the speed by loading the entire dataset trainloader into my GPU, instead of loading every batch separately. So, I would like to do something like
train_loader.to(CTX)

Is there an equivalent function for this? Because torch.utils.data.DataLoader does not have this attribute .to().
I work with an NVIDIA GeForce RTX 2060 with CUDA Toolkit 10.2 installed.

Output:
you can put your data of dataset in advance
train_dataset.train_data.to(CTX)  #train_dataset.train_data is a Tensor(input data)
train_dataset.train_labels.to(CTX)

for example of minst
import torch
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
batch_size = 64
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.1307,), (0.3081,))
])
train_data = datasets.MNIST(
    root='./dataset/minst/',
    train=True,
    download=False,
    transform=transform
)
train_loader = DataLoader(
    dataset=train_data,
    shuffle=True,
    batch_size=batch_size
)
train_data.train_data.to(torch.device("cuda:0"))  # put data into GPU entirely
train_data.train_labels.to(torch.device("cuda:0"))

I got this solution by using debugger...


Instruction: How can I fold a Tensor that I unfolded with PyTorch that has overlap?

Input:
I have a Tensor of size: torch.Size([1, 63840]) which I then unrolled:
inp_unfolded = inp_seq.unfold(1, 160, 80)

that gives me a shape of: torch.Size([1, 797, 160])
How can I re-fold that to get a Tensor of torch.Size([1, 63840])?

Output:
Well, actually the conditions, given t.unfold(i, n, s) are:

n >= s (otherwise step is skipping some original data and we cannot restore it)
n + s <= t.shape[i]

Then we can do it via:
def roll(x, n, s, axis=1):
    return torch.cat((p[0], p[1:][:, n-s:].flatten()), axis)

explanation:
p[0] is the starting chunk that is always unique at start
p[1:][:, n-s:] - then, we take rest of rolls and n-s depict how many elements will overlap between rolls so we want to ignore them and take only those from n-s
ilustration:
x.unfold(0, 5, 2)
tensor([[ 1.,  2.,  3.,  4.,  5.],
        [ 3.,  4.,  5.,  6.,  7.], # 3, 4, 5 are repeated
        [ 5.,  6.,  7.,  8.,  9.], # 5, 6, 7 are repeated...
        [ 7.,  8.,  9., 10., 11.],
        [ 9., 10., 11., 12., 13.],
        [11., 12., 13., 14., 15.],
        [13., 14., 15., 16., 17.]])

example:
>> x = torch.arange(1., 18)
>> p = x.unfold(0, 5, 2)
>> roll(p, 5, 2, 0)

tensor([ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10., 11., 12., 13., 14.,
        15., 16., 17.])

you can also try it with
x = torch.arange(1., 18).reshape(1, 17)

and axis 1


Instruction: How to train the Shared Layers in PyTorch

Input:
I have the follow code
import torch
import torch.nn as nn
from torchviz import make_dot, make_dot_from_trace

class Net(nn.Module):
    def __init__(self, input, output):
        super(Net, self).__init__()
        self.fc = nn.Linear(input, output)

    def forward(self, x):
        x = self.fc(x)
        x = self.fc(x)
        return x

model = Net(12, 12)
print(model)

x = torch.rand(1, 12)
y = model(x)
make_dot(y, params = dict(model.named_parameters()))

Here I reuse the self.fc twice in the forward.
The computational graph is look

I am confused about the computational graph and,
I am curious how to train this model in back propagation? It seem for me the gradient will live in a loop forever. Thanks a lot.

Output:
There are no issues with your graph. You can train it the same way as any other feed-forward model.

Regarding looping: Since it is a directed acyclic graph, the are no actual loops (check out the arrow directions).
Regarding backprop: Let’s consider fc.bias parameter. Since you are reusing the same layer two times, the bias has two outgoing arrows (used in two places of your net). During backpropagation stage the direction is reversed: bias will get gradients from two places, and these gradients will add up.
Regarding the graph: An FC layer can be represented as this: Addmm(bias, x, T(weight), where T is transposing and Addmm is matrix multiplication plus adding a vector. So, you can see how data (weight, bias) is passed into functions (Addmm, T)

https://pytorch.org/docs/stable/generated/torch.addmm.html
https://pytorch.org/docs/stable/generated/torch.t.html


Instruction: pytorch torch segmentation model syntax error in jupyter notebook

Input:
i get the error:
nn.MaxPool2d(kernel_size=2, stride=2),
^
SyntaxError: invalid syntax
with the following code:
import torch
import torch.nn as nn
import torch.nn.functional as F

class CNNSEG(nn.Module): # Define your model
    def __init__(self, num_classes=1):
        super(CNNSEG, self).__init__()
        #Adds one extra class to stand for the zero-padded pixels
        self.num_classes = num_classes + 1

        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.LocalResponseNorm(size=5, alpha=1e-4, beta=0.75),
        )

        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels=32, out_channels=256, kernel_size=5, stride=1, padding=2, groups=2),
            nn.ReLU(),j
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.LocalResponseNorm(size=5, alpha=1e-4, beta=0.75),
        )

        self.conv3 = nn.Sequential(
            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=5, stride=1, padding=2, groups=2),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.LocalResponseNorm(size=5, alpha=1e-4, beta=0.75),
        )

        self.score_conv = nn.Conv2d(in_channels=512, out_channels=num_classes, kernel_size=1, padding=0)
        self.deconv = nn.ConvTranspose2d(in_channels=num_classes, out_channels=num_classes,
            kernel_size=16, stride=8, bias=False)
        self.out_activation = nn.Softmax((num_classes, 4))

    def forward(self, x):
        out1 = self.conv1(x)
        out2 = self.conv2(out1)
        out3 = self.conv3(out2)
        out_score = self.score_conv(out3)
        out_deconv = self.deconv(out_score)
        return out_deconv

model = CNNSEG(num_classes=4) # We can now create a model using your defined segmentation model
print(model)

i haven't got any open brackets as far as I can see, I don't see what can be wrong here ? thanks

Output:
When setting the second layer (self.conv2) after the ReLu, you must have mistyped a "j".
That's why you get invalid syntax.


Instruction: Efficiently find the dot product of two lists of vectors stored as PyTorch tensors & preserve backprop

Input:
Suppose I had tensors X and Y which are both (batch_size, d) dimensional. I would like to find the (batch_size x 1) tensor resulting from [X[0]@Y[0].T, X[1]@Y[1].T, ...]
There are two ways I can think of doing this, neither of which are particularly efficient.
Way 1
product = torch.eye(batch_size) * X@Y.T
product = torch.sum(product, dim=1)

This works, but for large matrices there are a LOT of wasted computations
Way 2
product = torch.cat(
    [ X[i]@Y[i].T for i in X.size(0) ],
    dim=0
)

This is good in that no cycles are wasted, but it won't leverage any of the built-in parallelism torch offers.
I'm aware that numpy has a method that will do this, but converting the tensors to np arrays will destroy the chain of backpropagation, and this is for a neural net, so that's not an option.
Am I missing an obvious built in torch method, or am I stuck with these two options?

Output:
One way would be this. Simply use broadcasted matrix multiplication over reshaped row vectors of X and column vectors of Y.
import torch
X = X.reshape(batch_size, 1, d)
Y = Y.reshape(batch_size, d, 1)
product = torch.matmul(X, Y).squeeze(1)

The output product will have the required shape of (batch_size, 1) with the desired result.


Instruction: My PyTorch GAN is changing from producing random noise to darkness with no convergence. Why is this?

Input:
My code is very basic, just following the first lab from DeepLearning.ai's GAN specialization. However, my code does not have the same output, what is the reason for this. Sorry if this is just a silly mistake, this is my first experience with GANs. I begin by creating the Generator and Discriminator classes, my random noise function, and creating my models. I then run the training loop, but after 3 epochs, all of the outputs from the GAN are black.
import torch
from torch import nn
from tqdm.auto import tqdm
from torchvision import transforms
from torchvision.datasets import MNIST
from torchvision.utils import make_grid
from torch.utils.data import DataLoader, Dataset
import matplotlib.pyplot as plt

torch.manual_seed(0)

def show_tensor_images(image_tensor,num_images=25,size=(1,28,28)):
    image_unflat=image_tensor.detach().cpu().view(-1,*size)
    image_grid=make_grid(image_unflat[:num_images],nrow=5)
    plt.imshow(image_grid.permute(1,2,0).squeeze())
    plt.show()

class Generator(nn.Module):
    def __init__(self,z_dim):
        super(Generator,self).__init__()
        self.linear1=nn.Linear(z_dim,128)
        self.bn1=nn.BatchNorm1d(128)
        self.linear2=nn.Linear(128,256)
        self.bn2=nn.BatchNorm1d(256)
        self.linear3=nn.Linear(256,512)
        self.bn3=nn.BatchNorm1d(512)
        self.linear4=nn.Linear(512,1024)
        self.bn4=nn.BatchNorm1d(1024)
        self.linear5=nn.Linear(1024,784)
        self.relu=nn.ReLU(True)
        self.sigmoid=nn.Sigmoid()
    def forward(self,x):
        x=self.linear1(x)
        x=self.bn1(x)
        x=self.relu(x)
        x=self.linear2(x)
        x=self.bn2(x)
        x=self.relu(x)
        x=self.linear3(x)
        x=self.bn3(x)
        x=self.relu(x)
        x=self.linear4(x)
        x=self.bn4(x)
        x=self.relu(x)
        x=self.linear5(x)
        x=self.sigmoid(x)
        return(x)

def get_noise(n_samples,z_dim,device='cpu'):
    return torch.randn(n_samples,z_dim,device=device)

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator,self).__init__()
        self.linear1=nn.Linear(784,512)
        self.linear2=nn.Linear(512,256)
        self.linear3=nn.Linear(256,128)
        self.linear4=nn.Linear(128,1)
        self.relu=nn.LeakyReLU(0.2,True)
    def forward(self,x):
        x=self.linear1(x)
        x=self.relu(x)
        x=self.linear2(x)
        x=self.relu(x)
        x=self.linear3(x)
        x=self.relu(x)
        x=self.linear4(x)
        return(x)

criterion=nn.BCEWithLogitsLoss()
epochs=200
z_dim=64
display_step=500
batch_size=128
lr=0.00001
device='cuda'

dataloader=DataLoader(MNIST('.',download=True,transform=transforms.ToTensor()),batch_size=batch_size,shuffle=True)

gen=Generator(z_dim).to(device)
gen_opt=torch.optim.Adam(gen.parameters(),lr=lr)
disc=Discriminator().to(device)
disc_opt=torch.optim.Adam(disc.parameters(),lr=lr)

def get_disc_loss(gen,disc,criterion,real,num_images,z_dim,device):
    noise=get_noise(num_images,z_dim,device=device)
    gen_out=gen(noise)
    disc_fake_out=disc(gen_out.detach())
    fake_loss=criterion(disc_fake_out,torch.zeros_like(disc_fake_out))
    disc_real_out=disc(real)
    real_loss=criterion(disc_real_out,torch.zeros_like(disc_real_out))
    disc_loss=(fake_loss+real_loss)/2
    return(disc_loss)

def get_gen_loss(gen,disc,criterion,num_images,z_dim,device):
    noise=get_noise(num_images,z_dim,device=device)
    gen_out=gen(noise)
    disc_out=disc(gen_out)
    loss=criterion(disc_out,torch.ones_like(disc_out))
    return loss

cur_step=0
mean_generator_loss=0
mean_discriminator_loss=0
gen_loss=False
error=False
for epoch in range(epochs):
    for x,_ in tqdm(dataloader):
        cur_batch_size=len(x)
        x=x.view(cur_batch_size,-1).to(device)

        disc_opt.zero_grad()
        disc_loss=get_disc_loss(gen,disc,criterion,x,cur_batch_size,z_dim,device)
        disc_loss.backward(retain_graph=True)
        disc_opt.step()

        gen_opt.zero_grad()
        gen_loss=get_gen_loss(gen,disc,criterion,cur_batch_size,z_dim,device)
        gen_loss.backward()
        gen_opt.step()

        mean_discriminator_loss+=disc_loss.item()/display_step
        mean_generator_loss+=gen_loss.item()/display_step

        if cur_step%display_step==0 and cur_batch_size>0:
            print(f"Step {cur_step}: Generator loss: {mean_generator_loss}, discriminator loss: {mean_discriminator_loss}")
            fake_noise = get_noise(cur_batch_size, z_dim, device=device)
            fake = gen(fake_noise)
            show_tensor_images(fake)
            show_tensor_images(x)
            mean_generator_loss = 0
            mean_discriminator_loss = 0
        cur_step += 1

Output:
Your discriminator loss is wrong. The labels for the real images should be 1 instead of 0.
Updated code:
def get_disc_loss(gen,disc,criterion,real,num_images,z_dim,device):
    noise=get_noise(num_images,z_dim,device=device)
    gen_out=gen(noise)
    disc_fake_out=disc(gen_out.detach())
    fake_loss=criterion(disc_fake_out,torch.zeros_like(disc_fake_out))
    disc_real_out=disc(real)
    real_loss=criterion(disc_real_out,torch.ones_like(disc_real_out))
    disc_loss=(fake_loss+real_loss)/2
    return(disc_loss)

The output image looks pretty good to me:


Instruction: Trying to understand PyTorch SmoothL1Loss Implementation

Input:
I have been trying to go through all of the loss functions in PyTorch and build them from scratch to gain a better understanding of them and I’ve run into what is either an issue with my recreation, or an issue with PyTorch’s implementation.
According to Pytorch’s documentation for SmoothL1Loss it simply states that if the absolute value of the prediction minus the ground truth is less than beta, we use the top equation. Otherwise, we use the bottom one. Please see documentation for the equations.
Below is my implementation of this in the form of a minimum test:
import torch
import torch.nn as nn
import numpy as np

predictions = torch.randn(3, 5, requires_grad=True)
target = torch.randn(3, 5)

def l1_loss_smooth(predictions, targets, beta = 1.0):

    loss = 0
    for x, y in zip(predictions, targets):
        if abs(x-y).mean() < beta:
            loss += (0.5*(x-y)**2 / beta).mean()
        else:
            loss += (abs(x-y) - 0.5 * beta).mean()

    loss = loss/predictions.shape[0]

output = l1_loss_smooth(predictions, target)
print(output)
Gives an output of:
tensor(0.7475, grad_fn=<DivBackward0>)

Now the Pytorch implementation:
loss = nn.SmoothL1Loss(beta=1.0)
output = loss(predictions, target)
Gives an output of:
tensor(0.7603, grad_fn=<SmoothL1LossBackward>)

I can’t figure out where the error in implementation lies.
Upon looking a little deeper into the smooth_l1_loss function in the _C module (file: smooth_c_loss_op.cc) I noticed that the doc string mentions that it’s a variation on Huber Loss but the documentation for SmoothL1Loss says it is Huber Loss.
So overall, just confused on how it’s implemented and whether it’s a combo of SmoothL1Loss and Huber Loss, Just Huber Loss, or something else.

Output:
The description in the documentation is correct. Your implementation wrongly applies the case selection on the mean of the data. It should be an element-wise selection instead (if you think about the implementation of the vanilla L1 loss, and the motivation for smooth L1 loss).
The following code gives a consistent result:
import torch
import torch.nn as nn
import numpy as np

predictions = torch.randn(3, 5, requires_grad=True)
target = torch.randn(3, 5)

def l1_loss_smooth(predictions, targets, beta = 1.0):

    loss = 0

    diff = predictions-targets
    mask = (diff.abs() < beta)
    loss += mask * (0.5*diff**2 / beta)
    loss += (~mask) * (diff.abs() - 0.5*beta)

    return loss.mean()

output = l1_loss_smooth(predictions, target)
print(output)

loss = nn.SmoothL1Loss(beta=1.0)
output = loss(predictions, target)
print(output)


Instruction: calculate two losses in a model and backpropagate twice

Input:
I'm creating a model using BertModel to identify answer span (without using BertForQA).
I have an indepent linear layer for determining start and end token respectively. In init():
self.start_linear = nn.Linear(h, output_dim)

self.end_linear = nn.Linear(h, output_dim)

In forward(), I output a predicted start layer and predicted end layer:
 def forward(self, input_ids, attention_mask):

 outputs = self.bert(input_ids, attention_mask) # input = bert tokenizer encoding

 lhs = outputs.last_hidden_state # (batch_size, sequence_length, hidden_size)

 out = lhs[:, -1, :] # (batch_size, hidden_dim)

 st = self.start_linear(out)

 end = self.end_linear(out)

 predict_start = self.softmax(st)

 predict_end = self.softmax(end)

 return predict_start, predict_end

Then in train_epoch(), I tried to backpropagate the losses separately:
def train_epoch(model, train_loader, optimizer):

 model.train()

 total = 0

 st_loss, st_correct, st_total_loss = 0, 0, 0

 end_loss, end_correct, end_total_loss = 0, 0, 0

 for batch in train_loader:

   optimizer.zero_grad()

   input_ids = batch['input_ids'].to(device)

   attention_mask = batch['attention_mask'].to(device)

   start_idx = batch['start'].to(device)

   end_idx = batch['end'].to(device)

   start, end = model(input_ids=input_ids, attention_mask=attention_mask)

   st_loss = model.compute_loss(start, start_idx)

   end_loss = model.compute_loss(end, end_idx)

   st_total_loss += st_loss.item()

   end_total_loss += end_loss.item()

 # perform backward propagation to compute the gradients

   st_loss.backward()

   end_loss.backward()

 # update the weights

   optimizer.step()

But then I got on the line of end_loss.backward():
Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

Am I supposed to do the backward pass separately? Or should I do it in another way? Thank you!

Output:
The standard procedure is just to sum both losses and backpropagate on the sum.
It can be important to make sure both losses you want to sum have values that are on average approximately as big, or at least proportional to the importance you want each to have relative to one another(otherwise, the model is going to optimize for the bigger loss more than for the smaller one). In the span detection case, I'm guessing this won't be necessary however due to the apparent symmetry of the problem.


Instruction: Tensor format issue from converting Pytorch -> Onnx -> Tensorflow

Input:
I have an issue with Tensorflow model that is converted from Pytorch -> Onnx -> Tensorflow. The issue is the converted Tensorflow model expects the input in Pytorch format that is (batch size, number channels, height, width) but not in Tensorflow format (batch size, height, width, number channel). Therefore, I cannot use the model to process further with Vitis AI.
So I would like to ask is there is any ways to convert this Pytorch input format to Tensorflow format by using tools from Onnx, Tensorflow 1, or others?
My code is as below:
Pytorch -> Onnx
from hardnet import hardnet
import torch
import onnx

ckpt = torch.load('../hardnet.pth')
model_state_dict = ckpt['model_state_dict']
optimizer_state_dict = ckpt['optimizer_state_dict']

model = hardnet(11)
model.load_state_dict(model_state_dict)
model.eval()

dummy_input = torch.randn(1, 3, 1080, 1920)
input_names = ['input0']
output_names = ['output0']

output_file = 'hardnet.onnx'
torch.onnx.export(model, dummy_input, output_file, verbose=True,
    input_names=input_names, output_names=output_names,
    opset_version=11, keep_initializers_as_inputs=True)

onnx_model = onnx.load(output_file)
onnx.checker.check_model(onnx_model)
print('Passed Onnx')

Onnx -> Tensorflow 1 (using Tensorflow 1.15)
import cv2
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import onnx
from onnx_tf.backend import prepare

output_file = 'hardnet.onnx'
onnx_model = onnx.load(output_file)
output = prepare(onnx_model)
output.export_graph('hardnet.pb')
tf.compat.v1.disable_eager_execution()

def load_pb(path_to_pb: str):
    """From: https://stackoverflow.com/questions/51278213/what-is-the-use-of-a-pb-file-in-tensorflow-and-how-does-it-work
    """
    with tf.gfile.GFile(path_to_pb, "rb") as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
    with tf.Graph().as_default() as graph:
        tf.import_graph_def(graph_def, name='')
        return graph

graph = load_pb('hardnet.pb')
input = graph.get_tensor_by_name('input0:0')
output = graph.get_tensor_by_name('output0:0')
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
img = cv2.imread('train_0.jpg', cv2.IMREAD_COLOR)
img = cv2.resize(img, (1920,  1080))

img = img/255
img = img - mean
img = img/std
img = np.expand_dims(img, -1)
# To Pytorch format.
img = np.transpose(img, (3, 2, 0, 1))
img = img

with tf.Session(graph=graph) as sess:
    pred = sess.run(output, {input: img})

Output:
You could wrap your Pytorch model into another one that would do the transpose you want to have in TensorFlow. See the following example:
Let's say you have the following toy NN:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.rnn = nn.LSTM(10, 20, 2)

    def forward(self, x):
        h0 = torch.zeros(2, 3, 20)
        c0 = torch.zeros(2, 3, 20)
        return self.rnn(x, (h0, c0))

the exemplary pytorch/tensorflow input shape would be :
>> pytorch_input  = torch.randn(5, 3, 10)
>> tf_input  = torch.transpose(pytorch_input, 1, 2)

>> print("PyTorch input shape: ", pytorch_input.shape)
>> print("TensorFlow input shape: ", tf_input.shape)

PyTorch input shape:  torch.Size([5, 3, 10])
TensorFlow input shape:  torch.Size([5, 10, 3])

Now, the wrapper which will first transpose input and then pass transposed input to some model:
class NetTensorFlowWrapper(nn.Module):
    def __init__(self, main_module: nn.Module):
        super(NetTensorFlowWrapper, self).__init__()
        self.main_module = main_module

    def forward(self, x):
        x = torch.transpose(x, 1, 2)
        return self.main_module(x)

Then, this is possible:
net = Net()
net_wrapper = NetTensorFlowWrapper(net)

net(pytorch_input)
net_wrapper(tf_input)

and then, when you finally save your models like you did previously via torch.onnx.export and read their graph via onnx package (not torch.onnx) you will have...

for Net-  input 5x3x10 and no transpose layer

graph torch-jit-export (
  %input0[FLOAT, 5x3x10]

 {
  %76 = Shape(%input0)
  %77 = Constant[value = <Scalar Tensor []>]()

for NetTensorFlowWrapper-  input 5x10x3 and transpose layer

graph torch-jit-export (
  %input0[FLOAT, 5x10x3]

{
  %9 = Transpose[perm = [0, 2, 1]](%input0)
  %77 = Shape(%9)
  %78 = Constant[value = <Scalar Tensor []>]()
...


Instruction: Multi Head Attention: Correct implementation of Linear Transformations of Q, K, V

Input:
I am implementing the Multi-Head Self-Attention in Pytorch now. I looked at a couple of implementations and they seem a bit wrong, or at least I am not sure why it is done the way it is. They would often apply the linear projection just once:
    self.query_projection = nn.Linear(input_dim, output_dim)
    self.key_projection = nn.Linear(input_dim, output_dim)
    self.value_projection = nn.Linear(input_dim, output_dim)

and then they would often reshape the projection as
    query_heads = query_projected.view(batch_size, query_lenght, head_count, head_dimension).transpose(1,2)
    key_heads = key_projected.view(batch_size, key_len, head_count, head_dimension).transpose(1, 2)  # (batch_size, heads_count, key_len, d_head)
    value_heads = value_projected.view(batch_size, value_len, head_count, head_dimension).transpose(1, 2)  # (batch_size, heads_count, value_len, d_head)

    attention_weights = scaled_dot_product(query_heads, key_heads)

According to this code, each head will work on a piece of a projected query. However, the initial paper says that we need to have a different Linear projection for each head in the encoder.
Is this displayed implementation correct?

Output:
They are equivalent.
Theoretically (and in paper writing), it is easier to consider them as separate linear projections. Say if you have 8 heads, and each head has a M->N projection, then one would have 8 N by M matrix.
In implementation though, it is faster to have a M->8N transformation by having a 8N by M matrix.
One can concatenate the matrices in the first formulation to obtain the matrix in the second formulation.


Instruction: Pytorch RNN model not learning anything

Input:
Task: Predicting whether provided disaster tweets are real or not. Have already converted my textual data into tensors and then into train_loader. All the required code is mentioned below.
My Model Architecture
class RealOrFakeLSTM(nn.Module):

    def __init__(self, input_size, output_size, embedding_dim, hidden_dim, n_layers, bidirec, drop_prob):
        super().__init__()
        self.output_size=output_size
        self.n_layers=n_layers
        self.hidden_dim=hidden_dim
        self.bidirec=True;
        self.embedding=nn.Embedding(vocab_size, embedding_dim)
        self.lstm1=nn.LSTM(embedding_dim, hidden_dim, n_layers, dropout=drop_prob, batch_first=True, bidirectional=bidirec)
        #self.lstm2=nn.LSTM(hidden_dim, hidden_dim, n_layers, dropout=drop_prob, batch_first=True)
        self.dropout=nn.Dropout(drop_prob)
        self.fc=nn.Linear(hidden_dim, output_size)
        self.sigmoid=nn.Sigmoid()

    def forward(self, x):
        batch=len(x)
        hidden1=self.init_hidden(batch)
        #hidden2=self.init_hidden(batch)
        embedd=self.embedding(x)
        lstm_out1, hidden1=self.lstm1(embedd, hidden1)
        #lstm_out2, hidden2=self.lstm2(lstm_out1, hidden2)
        lstm_out1=lstm_out1.contiguous().view(-1, self.hidden_dim) # make it lstm_out2, if you un comment the other lstm cell.
        out=self.dropout(lstm_out1)
        out=self.fc(out)
        sig_out=self.sigmoid(out)
        sig_out=sig_out.view(batch, -1)
        sig_out=sig_out[:, -1]
        return sig_out

    def init_hidden(self, batch):
        if (train_on_gpu):
          if self.bidirec==True:
            hidden=(torch.zeros(self.n_layers*2, batch, self.hidden_dim).cuda(),torch.zeros(self.n_layers*2, batch, self.hidden_dim).cuda())
          else:
            hidden=(torch.zeros(self.n_layers, batch, self.hidden_dim).cuda(),torch.zeros(self.n_layers, batch, self.hidden_dim).cuda())
        else:
          if self.bidirec==True:
            hidden=(torch.zeros(self.n_layers*2, batch, self.hidden_dim),torch.zeros(self.n_layers*2, batch, self.hidden_dim))
          else:
            hidden=(torch.zeros(self.n_layers, batch, self.hidden_dim),torch.zeros(self.n_layers, batch, self.hidden_dim))
        return hidden

Hyper parameters and training
learning_rate=0.005
epochs=50
vocab_size = len(vocab_to_int)+1 # +1 for the 0 padding
output_size = 2
embedding_dim = 300
hidden_dim = 256
n_layers = 2
batch_size=23
net=RealOrFakeLSTM(vocab_size, output_size, embedding_dim, hidden_dim, n_layers, True, 0.3)
net.to(device)
criterion=nn.BCELoss()
optimizer=torch.optim.Adam(net.parameters(),lr=learning_rate)
net.train()
loss_arr=np.array([])
lossPerEpoch=np.array([])
for i in range(epochs):
  total_loss=0;
  for input,label in train_loader:
    if train_on_gpu:
      input=input.to(device)
      label=label.to(device)
    optimizer.zero_grad()
    input=input.clone().detach().long()
    out=net(input)
    loss=criterion(out.squeeze(),label.float())
    loss_arr=np.append(loss_arr,loss.cpu().detach().numpy())
    loss.backward()
    optimizer.step()
    total_loss+=loss
  total_loss=total_loss/len(train_loader)
  lossPerEpoch=np.append(lossPerEpoch,total_loss.cpu().detach().numpy())
  print("Epoch ",i,": ",total_loss)
  torch.save(net.state_dict(), Path+"/RealOrFakeLSTM.pt")
  torch.save(net, Path+"/RealOrFakeLSTM.pth")
current_time=str(time.time())
torch.save(net.state_dict(), Path+"/pt/RealOrFakeLSTM"+'_pt_'+current_time+".pt")
torch.save(net, Path+"/pth/RealOrFakeLSTM"+'_pth_'+current_time+".pth")

The total loss values are all almost same, All the outcomes probabilities in the test dataset are exactly same. I am quite new to this, so hyper parameter tuning, i am kinda going with bruteforce, but nothing seems to work, I think my problem is not with the architecture but with the training part, as all the predictions are exactly same.

Output: From what I can tell you are initializing the hidden1=self.init_hidden(batch) in every forward pass. That should not be correct. Initializing a layer in every forward pass explains the behavior you described.


Instruction: Classes in Coco dataset

Input:
I have been checking out this detr repository and the total number of classes are 100, but 10 of these are empty string as shown here.
Is there any particular reason behind this?

Output:
Basically, the COCO dataset was described in a paper before its release (you can find it here). At this point, the authors gave a list of the 91 types of objects that would be in the dataset.
But when the 2014 and 2017 datasets sere released, it turned out that you could find only 80 of these objects in the annotations.
The list you have is the original list of objects (as described in the paper) but with every object that does not appear in the 2014 and 2017 replaced by the empty string "".
My guess is that the sole purpose of keeping these "phantom" objects is to keep consistency with object ids that may have been fixed someday in the past.
If you want to learn more about it, you can look at this blog entry.


Instruction: How to use different optimizers for each model layer Pytorch?

Input: I have a two layer network built in pytorch and two two different optimizers. I would like to use one optimizer on the first layer and the other optimizers on the second layer. Is this possible?

Output:
Yes this is possible: When initializing an optimizer you need to pass it the parameters that you want to optimize which is where you have to do this division. For instance:
import torch.nn as nn
import torch.optim

net = nn.Sequential(
    nn.Linear(1, 3),
    nn.Linear(3, 5),
    nn.Linear(5, 1)
)

opt1 = torch.optim.Adam(params=net[0].parameters(), lr=0.1)
opt2 = torch.optim.Adam(params=[*net[1].parameters(), *net[2].parameters()], lr=0.001)


Instruction: How to check if a model is in train or eval mode in Pytorch?

Input: How to check from within a model if it is currently in train or eval mode?

Output:
From the Pytorch forum, with a small tweak:
use
if self.training:
    # it's in train mode
else:
    # it's in eval mode

Always better to have a stack overflow answer than to look at forums.
Explanation about the modes


Instruction: Getting range of values from Pytorch Tensor

Input:
I am trying to get a specific range of values from my pytorch tensor.
tensor=torch.tensor([0,1,2,3,4,5,6,7,8,9])
new_tensor=tensor[tensor>2]
print(new_tensor)

This will give me a tensor with scalars of 3-9
new_tensor2=tensor[tensor<8]
print(new_tensor2)

This will give me a tensor with scalars of 0-7
new_tensor3=tensor[tensor>2 and tensor<8]
print(new_tensor3)

However this raises an error. Would I be able to get a tensor with the values of 3-7 using something like this? I am trying to edit the tensor directly, and do not wish to change the order of the tensor itself.
grad[x<-3]=0.1
grad[x>2]=1
grad[(x>=-3 and x<=2)]=siglrelu(grad[(x>=-3 and x<=2)])*(1.0-siglrelu(grad[(x>=-3 and x<=2)]))

This is what I am really going for, and I am not exactly sure of how to go about this. Any help is appreciated, thank you!

Output:
You can use & operation,
t = torch.arange(0., 10)
print(t)
print(t[(t > 2) & (t < 8)])

Output is,
tensor([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])
tensor([3., 4., 5., 6., 7.])


Instruction: How to convert tensorflow.js model weights to pytorch tensors, and back?

Input:
I am using ml5.js, a wrapper around tensorflowjs. I want to train a neural network in the browser, download the weights, process them as tensors in pyTorch, and load them back into the browser's tensorflowjs model. How do I convert between these formats tfjs <-> pytorch?
The browser model has a save() function which generates three files. A metadata file specific to ml5.js (json), a topology file describing model architecture (json), and a binary weights file (bin).
// Browser
model.save()

// HTTP/Download
model_meta.json   (needed by ml5.js)
model.json        (needed by tfjs)
model.weights.bin (needed by tfjs)

# python backend
import json

with open('model.weights.bin', 'rb') as weights_file:
    with open('model.json', 'rb') as model_file:
        weights = weights_file.read()
        model = json.loads(model_file.read())
        ####
        pytorch_tensor = convert2tensor(weights, model) # whats in this function?
        ####
        # Do some processing in pytorch

        ####
        new_weights_bin = convert2bin(pytorch_tensor, model) # and in this?
        ####

Here is sample javascript code to generate and load the 3 files in the browser. To load, select all 3 files at once in the dialog box. If they are correct, a popup will show a sample prediction.

Output:
I was able to find a way to convert from tfjs model.weights.bin to numpy's ndarrays. It is trivial to convert from numpy arrays to pytorch state_dict which is a dictionary of tensors and their names.
Theory
First, the tfjs representation of the model should be understood. model.json describes the model. In python, it can be read as a dictionary. It has the following keys:

The model architecture is described as another json/dictionary under the key modelTopology.

It also has a json/dictionary under the key weightsManifest which describes the type/shape/location of each weight wrapped up in the corresponding  model.weights.bin file. As an aside, the weights manifest allows for multiple .bin files to store weights.

Tensorflow.js has a companion python package tensorflowjs, which comes with utility functions to read and write weights between the tf.js binary and numpy array format.
Each weight file is read as a "group". A group is a list of dictionaries with keys name and data which refer to the weight name and the numpy array containing weights. There are optionally other keys too.
group = [{'name': weight_name, 'data': np.ndarray}, ...]   # 1 *.bin file

Application
Install tensorflowjs. Unfortunately, it will also install tensorflow.
pip install tensorflowjs

Use these functions. Note that I changed the signatures for convenience.
from typing import Dict, ByteString
import torch
from tensorflowjs.read_weights import decode_weights
from tensorflowjs.write_weights import write_weights

def convert2tensor(weights: ByteString, model: Dict) -> Dict[str, torch.Tensor]:
    manifest = model['weightsManifest']
    # If flatten=False, returns a list of groups equal to the number of .bin files.
    # Use flatten=True to convert to a single group
    group = decode_weights(manifest, weights, flatten=True)
    # Convert dicts in tfjs group format into pytorch's state_dict format:
    # {name: str, data: ndarray} -> {name: tensor}
    state_dict = {d['name']: torch.from_numpy(d['data']) for d in group}
    return state_dict

def convert2bin(state_dict: Dict[str: np.ndarray], model: Dict, directory='./'):
    # convert state_dict to groups (list of 1 group)
    groups = [[{'name': key, 'data': value} for key, value in state_dict.items()]]
    # this library function will write to .bin file[s], but you can read it back
    # or change the function internals my copying them from source
    write_weights(groups, directory, write_manifest=False)


Instruction: PyTorch : Aggregate two models

Input:
Hello and greetings from Greece
class Model(nn.Module):

def __init__(self, embedding_size, num_numerical_cols, output_size, layers, p=0.4):
    super().__init__()
    self.all_embeddings = nn.ModuleList([nn.Embedding(ni, nf) for ni, nf in embedding_size])
    self.embedding_dropout = nn.Dropout(p)
    self.batch_norm_num = nn.BatchNorm1d(num_numerical_cols)

    all_layers = []
    num_categorical_cols = sum((nf for ni, nf in embedding_size))
    input_size = num_categorical_cols + num_numerical_cols

    for i in layers:
        all_layers.append(nn.Linear(input_size, i))
        all_layers.append(nn.ReLU(inplace=True))
        all_layers.append(nn.BatchNorm1d(i))
        all_layers.append(nn.Dropout(p))
        input_size = i

    all_layers.append(nn.Linear(layers[-1], output_size))

    self.layers = nn.Sequential(*all_layers)

def forward(self, x_categorical, x_numerical):
    embeddings = []
    for i,e in enumerate(self.all_embeddings):
        embeddings.append(e(x_categorical[:,i]))
    x = torch.cat(embeddings, 1)
    x = self.embedding_dropout(x)

    x_numerical = self.batch_norm_num(x_numerical)
    x = torch.cat([x, x_numerical], 1)
    x = self.layers(x)
    return x

Suppose I have this nn for classification and I create two instances
model_1=Model(categorical_embedding_sizes, numerical_data.shape[1], 2, [200,100,50], p=0.4)
model_2=Model(categorical_embedding_sizes, numerical_data.shape[1], 2, [200,100,50], p=0.4)

Αnd after I trained these two models  i saved them with torch.save as model_1.pt and model_2.pt
Is there a way to create a new model with the mean parameters of the two models ?
something like
model_new.weight=(model_1.weight+model_2.weight)/2
model_new.bias=(model_1.bias+model_2.bias)/2

Thank you in advance

Output:
You can easily do this by generating a state dictionary from your two models' state dictionaries:
state_1 = model_1.state_dict()
state_2 = model_2.state_dict()

for layer in state_1:
    state_1[layer] = (state_1[layer] + state_2[layer])/2

The above will loop through parameters (weights and biases) of all layers.
Then overwrite this new state on either model_1 or a newly instanced model, like so:
model_new = Model(categorical_embedding_sizes, numerical_data.shape[1], 2, [200,100,50], p=0.4)
model_new.load_state_dict(state1)


Instruction: How to check if a small tensor is inside anthor large tensor

Input:
a = torch.tensor([[1,1],[2,2]])

I want to know if tensor([1,1]) is inside in a (return one bool)
a.eq(torch.tensor([1,1]))
tensor([[ True, True],
[False, False]])

–> which should return a True to my case.
a.eq(torch.tensor([1,2]))
tensor([[ True, False],
[False, True]])

–> which should return a false to my case.
Any suggestion?

Output:
You could convert it to a list and then check via in operator:
p = torch.arange(1, 5).reshape((2,2))
k = torch.tensor([1,2])
v = torch.tensor([1,3])

>> k.tolist() in p.tolist()

True

>> v.tolist() in p.tolist()

False

Or if you want to do it via torch only
torch.count_nonzero(k==p, 1) == len(k)


Instruction: Pytorch how to reshape/reduce the number of filters without altering the shape of the individual filters

Input:
With a 3D tensor of shape (number of filters, height, width), how can one reduce the number of filters with a reshape which keeps the original filters together as whole blocks?
Assume the new size has dimensions chosen such that a whole number of the original filters can fit side by side in one of the new filters. So an original size of (4, 2, 2) can be reshaped to (2, 2, 4).
A visual explanation of the side by side reshape where you see the standard reshape will alter the individual filter shapes:

I have tried various pytorch functions such as gather and select_index but not found a way to get to the end result in a general manner (i.e. works for different numbers of filters and different filter sizes).
I think it would be easier to rearrange the tensor values after performing the reshape but could not get a tensor of the pytorch reshaped form:
[[[1,2,3,4],
  [5,6,7,8]],

 [[9,10,11,12],
  [13,14,15,16]]]

to:
[[[1,2,5,6],
  [3,4,7,8]],

 [[9,10,13,14],
  [11,12,15,16]]]

for completeness, the original tensor before reshaping:
[[[1,2],
  [3,4]],

 [[5,6],
  [7,8]],

 [[9,10],
  [11,12]],

 [[13,14],
  [15,16]]]

Output:
Another option is to construct a list of parts and concatenate them
x = torch.arange(4).reshape(4, 1, 1).repeat(1, 2, 2)
y = torch.cat([x[i::2] for i in range(2)], dim=2)

print('Before\n', x)
print('After\n', y)

which gives
Before
 tensor([[[0, 0],
         [0, 0]],

        [[1, 1],
         [1, 1]],

        [[2, 2],
         [2, 2]],

        [[3, 3],
         [3, 3]]])
After
 tensor([[[0, 0, 1, 1],
         [0, 0, 1, 1]],

        [[2, 2, 3, 3],
         [2, 2, 3, 3]]])

Or a little more generally we could write a function that takes groups of neighbors along a source dimension and concatenates them along a destination dimension
def group_neighbors(x, group_size, src_dim, dst_dim):
    assert x.shape[src_dim] % group_size == 0
    return torch.cat([x[[slice(None)] * (src_dim) + [slice(i, None, group_size)] + [slice(None)] * (len(x.shape) - (src_dim + 2))] for i in range(group_size)], dim=dst_dim)

x = torch.arange(4).reshape(4, 1, 1).repeat(1, 2, 2)
# read as "take neighbors in groups of 2 from dimension 0 and concatenate them in dimension 2"
y = group_neighbors(x, group_size=2, src_dim=0, dst_dim=2)

print('Before\n', x)
print('After\n', y)


Instruction: PyTorch with yolov5: color channel and result display

Input:
I have a script that grabs an application's screenshot and displays it. it works quite nicely on my machine like a video with around 60FPS. Now I want to use a yolov5 object detection model on these frames, with TorchHub, as advised here.
The following works:
import os
os.getcwd()
from PIL import ImageGrab
import numpy as np
import cv2
import pyautogui
import win32gui
import time
from mss import mss
from PIL import Image
import tempfile
os.system('calc')
sct = mss()
xx=1
tstart = time.time()
while xx<10000:
    hwnd = win32gui.FindWindow(None, 'Calculator')
    left_x, top_y, right_x, bottom_y = win32gui.GetWindowRect(hwnd)
    #screen = np.array(ImageGrab.grab( bbox = (left_x, top_y, right_x, bottom_y ) ) )
    bbox = {'top': top_y, 'left': left_x, 'width': right_x-left_x, 'height':bottom_y-top_y }
    screen = sct.grab(bbox)
    scr = np.array(screen)

    cv2.imshow('window', scr)
    if cv2.waitKey(25) & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        break
    xx+=1
cv2.destroyAllWindows()
tend = time.time()
print(xx/(tend-tstart))
print((tend-tstart))
os.system('taskkill /f /im calculator.exe')

Below I try to import torch and use my previously trained model,
screen = sct.grab(bbox)
scr = np.array(screen)
result = model(scr, size=400)
result.save("test.png") #this gives a TypeError: save() takes 1 positional argument but 2 were given
result.show() #this opens a new Paint instance for every frame instead of keeping the same window.
# The shown image is also in a wrong color channel
scr = cv2.imread("test.png")
# How can I use the `result` as argument to cv2.imshow(),
# without saving to disk if possible?

My questions:

result.show() shows an image with wrong color channel compared to cv2.imshow(), how can I ensure that the image being fed to model is on the correct channel?
The performance of classification and detection drastically decrease compared to the training validation, perhaps because of 1?
Do you know how I can display the result model image with bounding boxes in a single window like what cv2.imshow() does ? (result.show() opens a new Paint process instance for each frame) ? How can I save this result image to disk and find more documentation on how to interact with model objects in general?

Output:
The following worked: result = model(cv2.cvtColor(scr, cv2.COLOR_BGR2RGB), size=400)
This solved the accuracy problem and model.save() has pre-defined output names  which are not currently changeable, it takes no arguments.
model.show() shows the correct color channel output when fed the correct color channel as input.


Instruction: What parameters do I change to train a pytorch model from scratch?

Input:
I followed this tutorial to train a pytorch model for instance segmentation:
https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html
I would not like to train a model on entirely different data and classes, totally unrelated to COCO. What changes do I need to make to retrain the model. From my reading I'm guessing besides have the correct number of classes I just need to train this line:
model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=True)

to
model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=False)

But I notice there is another parameters: pretrained_backbone=True, trainable_backbone_layers=None should they be changed too?

Output:
The function signature is

torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=False, progress=True, num_classes=91, pretrained_backbone=True, trainable_backbone_layers=3, **kwargs)

Setting pretrained=False will tell PyTorch not to download model pre-trained on COCO train2017. You want it as you're interested in training.
Usually, this is enough if you want to train on a different dataset.
When you set pretrained=False, PyTorch will download pretrained ResNet50 on ImageNet. And by default, it'll freeze first two blocks named conv1 and layer1. This is how it was done in Faster R-CNN paper which frooze the initial layers of pretrained backbone.
(Just print model to check its structure).
layers_to_train = ['layer4', 'layer3', 'layer2', 'layer1', 'conv1'][:trainable_layers]

Now, if you don't even want the first two layers to freeze, you can set trainable_backbone_layers=5 (done automatically, when you set pretrained_backbone=False), which will train the entire resnet backbone from scratch.
Check PR#2160.


Instruction: (Numpy or PyTorch) Sum array elements for given bins

Input:
I would like this problem to be solved using PyTorch tensors. If there is no efficient solution in torch, then feel free to suggest a numpy solution.
Let a be a 1-dimensional tensor (or numpy array), and bin_indices be a tensor (np array) of integers between 0 and n excluded. I want to compute the array bins that at position i contains the sum of elements of a[bins_indices == i].
n = 3

a = [1, 4, 3, -2, 5]  # Values

bins_indices = [0, 0, 1, 2, 0]  # Correspondent bin indices

bins = [10, 3, -2]  # bins[0] = 1 + 4 + 5 etc. bins has 3 elements since n=3

If you can provide also a way of making this work for batches I would be immensely grateful to you!

Output:
Here's a one-line Numpy solution I could think of:
bins = [np.sum(a[np.argwhere(bins_indices == i).flatten()]) for i in range(len(a))]


Instruction: No model named torch

Input:
Dear fellow humans and others,
i have a question about using and importing pytorch into my editor.
I installed pytorch through Conda and in my terminal editor I can use pytorch,see image but not in Visual studio code or Pycharm!
No model named torch error code in VSC
Why is that?
Very sorry for the basic question and thank you for sparing some time!
Should I use pip?

Output:
Reason: The environment where the module "torch" is stored is not the environment you currently select in VSCode.
For installing and using the module "torch" in VSCode, you could refer to the following:

Check the environment. The Python environment displayed in the lower left corner of VSCode is the same as that of the terminal.

Install the module. (pip install torch)

Run.

Check the installation of the module. (pip show torch)

Reference: Environment in VSCode.


Instruction: How to check if a tensor is on cuda or send it to cuda in Pytorch?

Input:
I have a tensor
t = torch.zeros((4, 5, 6))
How to check if it is on gpu or not, and send it to gpu and back?

Output:
From the pytorch forum
use t.is_cuda, t.cuda(), t.cpu()
t = torch.randn(2,2)
t.is_cuda  # returns False
t = torch.randn(2,2).cuda()
t.is_cuda  # returns True
t = t.cpu()
t.is_cuda  # returns False

When passing to and from gpu and cpu, new arrays are allocated on the relevant device.


Instruction: How can I crop away a tensor’s constant value padding (padding height and width are the same) with an unknown value and size?

Input:
How can I crop away a tensor’s constant value padding (padding height and width are the same) with an unknown value and size?
I would think that because the padding surrounding my tensor has a constant value, and the same height / width, that it should be possible to know where to crop the tensor to remove the padding.
import torch

# Test tensor with NCHW dimensions
a = torch.randn(1,4,5,5) # Can have any H & W size

# Create padding for testing
b = torch.nn.functional.pad(a, (2,2,2,2), 'constant', value=1.2) # Value can be any number

c = # equal to a, without being able to use the variables a or b (or their argument values)

NumPy solutions are acceptable as I can easily convert them to PyTorch.
Edit:
pad = torch.where(b[0, 0] - b[0, 0, 0, 0] != 0)[0][0]

x_pad, y_pad = 0, 0
if (b.size(3) % 2) == 0:
    x_pad = 1
if (b.size(2) % 2) == 0:
    y_pad = 1
c = b[:, :, pad : -(pad + y_pad), pad : -(pad + x_pad)]

assert c == a

Output:
You can get an idea of the content of a feature map by taking its middle row and measure the padding by looking for the first element change:
midrow = b[0, 0, b.shape[3]//2, :]
pad = (midrow[:-1] == midrow[:1])[:midrow.shape[0]//2].sum()

Alternatively you could substract one of the feature maps with the padding value and find the first non zero value, which would be the padding size:
pad = torch.where(b[0,0] - b[0,0,0,0] != 0)[0][0]

Having the padding, we can discard the right amount of values around the feature maps for all batch elements and all channels:
a = b[:, :, pad:-pad, pad:-pad]