Instruction: If we combine one trainable parameters with a non-trainable parameter, is the original trainable param trainable?

Input:
I have two nets and I combine their parameters in some fancy way using only pytorch operations. I store the result in a third net which has its parameters set to non-trainable. Then I proceed and pass data through this new net. The new net is just a placeholder for:

placeholder_net.W = Op( not_trainable_net.W, trainable_net.W )

Then I pass data:

output = placeholder_net(input)

I am concerned that since the parameters of the placeholder net are set to non-trainable that it won’t actually train the variable that it should train. Will this happen? Or what is the result when you combine a trainable param with and non-trainable param (and then set that where the param is not trainable)?

Current solution:

del net3.conv0.weight
net3.conv0.weight = net.conv0.weight + net2.conv0.weight

import torch
from torch import nn
import torch.optim as optim

import torchvision
import torchvision.transforms as transforms

from collections import OrderedDict

import copy

def dont_train(net):
    '''
    set training parameters to false.
    '''
    for param in net.parameters():
        param.requires_grad = False
    return net

def get_cifar10():
    transform = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
    trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,shuffle=True, num_workers=2)
    classes = ('plane', 'car', 'bird', 'cat','deer', 'dog', 'frog', 'horse', 'ship', 'truck')
    return trainloader,classes

def combine_nets(net_train, net_no_train, net_place_holder):
        '''
            Combine nets in a way train net is trainable
        '''
        params_train = net_train.named_parameters()
        dict_params_place_holder = dict(net_place_holder.named_parameters())
        dict_params_no_train = dict(net_no_train.named_parameters())
        for name, param_train in params_train:
            if name in dict_params_place_holder:
                layer_name, param_name = name.split('.')
                param_no_train = dict_params_no_train[name]
                ## get place holder layer
                layer_place_holder = getattr(net_place_holder, layer_name)
                delattr(layer_place_holder, param_name)
                ## get new param
                W_new = param_train + param_no_train  # notice addition is just chosen for the sake of an example
                ## store param in placehoder net
                setattr(layer_place_holder, param_name, W_new)
        return net_place_holder

def combining_nets_lead_to_error():
    '''
    Intention is to only train the net with trainable params.
    Placeholder rnet is a dummy net, it doesn't actually do anything except hold the combination of params and its the
    net that does the forward pass on the data.
    '''
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    ''' create three musketeers '''
    net_train = nn.Sequential(OrderedDict([
          ('conv1', nn.Conv2d(1,20,5)),
          ('relu1', nn.ReLU()),
          ('conv2', nn.Conv2d(20,64,5)),
          ('relu2', nn.ReLU())
        ])).to(device)
    net_no_train = copy.deepcopy(net_train).to(device)
    net_place_holder = copy.deepcopy(net_train).to(device)
    ''' prepare train, hyperparams '''
    trainloader,classes = get_cifar10()
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net_train.parameters(), lr=0.001, momentum=0.9)
    ''' train '''
    net_train.train()
    net_no_train.eval()
    net_place_holder.eval()
    for epoch in range(2):  # loop over the dataset multiple times
        running_loss = 0.0
        for i, (inputs, labels) in enumerate(trainloader, 0):
            optimizer.zero_grad() # zero the parameter gradients
            inputs, labels = inputs.to(device), labels.to(device)
            # combine nets
            net_place_holder = combine_nets(net_train,net_no_train,net_place_holder)
            #
            outputs = net_place_holder(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            # print statistics
            running_loss += loss.item()
            if i % 2000 == 1999:  # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0
    ''' DONE '''
    print('Done \a')

if __name__ == '__main__':
    combining_nets_lead_to_error()

Output:
First, do not use eval() mode for any network. Set requires_grad flag to false to make the parameters non-trainable for only the second network and train the placeholder network.

If this doesn't work, you can try the following approach which I prefer.

Instead of using multiple networks, you can use a single network and use a non-trainable layer as a parallel connection after every trainable layer before non-linearity.

For example look at this image:

Set requires_grad flag to false to make the parameters non-trainable. Do not use eval() and train the network.

Combining outputs of the layers before non-linearity is important. Initialize the parameters of the parallel layer and choose the post-operation such that it gives the same result as when you combine the parameters.


Instruction: Any example of torch 0.4.0 nn.LayerNorm example for nn.LSTMCell?

Input:
In pytorch 0.4.0 release, there is a nn.LayerNorm module.

I want to implement this layer to my LSTM network, though I cannot find any implementation example on LSTM network yet.

And the pytorch Contributor implies that this nn.LayerNorm is only applicable through nn.LSTMCells.

It will be a great help if I can get any git repo or some code that implements nn.LayerNorm on nn.LSTMcell or any torch LSTM network.

Thanks in advance

Output:
I am also looking for a solution. Here is an example from https://github.com/pytorch/pytorch/issues/11335
Thanks to @jinserk

class LayerNormLSTMCell(nn.LSTMCell):

def __init__(self, input_size, hidden_size, bias=True):
    super().__init__(input_size, hidden_size, bias)

    self.ln_ih = nn.LayerNorm(4 * hidden_size)
    self.ln_hh = nn.LayerNorm(4 * hidden_size)
    self.ln_ho = nn.LayerNorm(hidden_size)

def forward(self, input, hidden=None):
    self.check_forward_input(input)
    if hidden is None:
        hx = input.new_zeros(input.size(0), self.hidden_size, requires_grad=False)
        cx = input.new_zeros(input.size(0), self.hidden_size, requires_grad=False)
    else:
        hx, cx = hidden
    self.check_forward_hidden(input, hx, '[0]')
    self.check_forward_hidden(input, cx, '[1]')

    gates = self.ln_ih(F.linear(input, self.weight_ih, self.bias_ih)) \
             + self.ln_hh(F.linear(hx, self.weight_hh, self.bias_hh))
    i, f, o = gates[:, :(3 * self.hidden_size)].sigmoid().chunk(3, 1)
    g = gates[:, (3 * self.hidden_size):].tanh()

    cy = (f * cx) + (i * g)
    hy = o * self.ln_ho(cy).tanh()
    return hy, cy

class LayerNormLSTM(nn.Module):

def __init__(self, input_size, hidden_size, num_layers=1, bias=True, bidirectional=False):
    super().__init__()
    self.input_size = input_size
    self.hidden_size = hidden_size
    self.num_layers = num_layers
    self.bidirectional = bidirectional

    num_directions = 2 if bidirectional else 1
    self.hidden0 = nn.ModuleList([
        LayerNormLSTMCell(input_size=(input_size if layer == 0 else hidden_size * num_directions),
                          hidden_size=hidden_size, bias=bias)
        for layer in range(num_layers)
    ])

    if self.bidirectional:
        self.hidden1 = nn.ModuleList([
            LayerNormLSTMCell(input_size=(input_size if layer == 0 else hidden_size * num_directions),
                              hidden_size=hidden_size, bias=bias)
            for layer in range(num_layers)
        ])

def forward(self, input, hidden=None):
    seq_len, batch_size, hidden_size = input.size()  # supports TxNxH only
    num_directions = 2 if self.bidirectional else 1
    if hidden is None:
        hx = input.new_zeros(self.num_layers * num_directions, batch_size, self.hidden_size, requires_grad=False)
        cx = input.new_zeros(self.num_layers * num_directions, batch_size, self.hidden_size, requires_grad=False)
    else:
        hx, cx = hidden

    ht = [[None, ] * (self.num_layers * num_directions)] * seq_len
    ct = [[None, ] * (self.num_layers * num_directions)] * seq_len

    if self.bidirectional:
        xs = input
        for l, (layer0, layer1) in enumerate(zip(self.hidden0, self.hidden1)):
            l0, l1 = 2 * l, 2 * l + 1
            h0, c0, h1, c1 = hx[l0], cx[l0], hx[l1], cx[l1]
            for t, (x0, x1) in enumerate(zip(xs, reversed(xs))):
                ht[t][l0], ct[t][l0] = layer0(x0, (h0, c0))
                h0, c0 = ht[t][l0], ct[t][l0]
                t = seq_len - 1 - t
                ht[t][l1], ct[t][l1] = layer1(x1, (h1, c1))
                h1, c1 = ht[t][l1], ct[t][l1]
            xs = [torch.cat((h[l0], h[l1]), dim=1) for h in ht]
        y  = torch.stack(xs)
        hy = torch.stack(ht[-1])
        cy = torch.stack(ct[-1])
    else:
        h, c = hx, cx
        for t, x in enumerate(input):
            for l, layer in enumerate(self.hidden0):
                ht[t][l], ct[t][l] = layer(x, (h[l], c[l]))
                x = ht[t][l]
            h, c = ht[t], ct[t]
        y  = torch.stack([h[-1] for h in ht])
        hy = torch.stack(ht[-1])
        cy = torch.stack(ct[-1])

    return y, (hy, cy)


Instruction: How to add a L2 regularization term in my loss function

Input:
I’m going to compare the difference between with and without regularization, so I want to custom two loss functions.

My loss function with L2 norm:

###NET
class CNN(nn.Module):
def __init__(self):
    super(CNN,self).__init__()
    self.layer1 = nn.Sequential(
        nn.Conv2d(3, 16, kernel_size = 5, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(2))
    self.layer2 = nn.Sequential(
        nn.Conv2d(16, 32, kernel_size = 5, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(2))
    self.layer3 = nn.Sequential(
        nn.Conv2d(32, 32, kernel_size = 5, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(4))
    self.fc = nn.Linear(32*32*32,11)
def forward(self, x):
    out = self.layer1(x)
    out = self.layer2(out)
    out = self.layer3(out)
    out = out.view(out.size(0), -1)
    out = self.fc(out)
    return out

net = CNN()

###OPTIMIZER
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr = LR, momentum = MOMENTUM)

  1.How can I add a L2 norm in my loss function?

  2.If I want to write the loss function by myself (without using optim.SGD) and do the grad-decent by autograd, how can I do?

Thanks for your help!

Output:
You can explicitly compute the norm of the weights yourself, and add it to the loss.

reg = 0
for param in CNN.parameters():
  reg += 0.5 * (param ** 2).sum()  # you can replace it with abs().sum() to get L1 regularization
loss = criterion(CNN(x), y) + reg_lambda * reg  # make the regularization part of the loss
loss.backward()  # continue as usuall

See this thread for more info.


Instruction: Anaconda / Pytorch - Getting error when trying to get Pytorch working

Input:
I've tried the installing Pytorch with Anaconda on a Windows 10 system but have gotten a variety of errors back with each attempt. I downloaded a windows compatible tar file of the package from https://anaconda.org/peterjc123/pytorch/files and was apparently able to get the the package installed using conda install filename.tar.bz2. By apparently, I mean that the Pytorch shows up in the list of packages available in the environment I set up for using Pytorch. But when I go to load the package I get this error:

>>> import torch
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\Users\conner\Anaconda3\envs\pytorch\lib\site-packages\torch\__init__.py", line 76, in <module>
    from torch._C import *
ImportError: DLL load failed: The specified module could not be found.

I also tried updating Anaconda with conda update --all which raised a relevant warning:

(pytorch) C:\Users\User 1\Downloads>conda update --all
Fetching package metadata .............
Solving package specifications:
Warning: ['Dependency missing in current win-64 channels: \n  - pytorch -> mkl >=2018'], skipping
Fetching package metadata .............
Solving package specifications:
Warning: ['Dependency missing in current win-64 channels: \n  - pytorch -> mkl >=2018'], skipping

NoPackagesFoundError: Dependency missing in current win-64 channels:
  - pytorch -> mkl >=2018

The mkl package also shows up in the list of packages connected to the pytorch environment. I interpret the error to mean that there isn't a recent enough version of mkl. Would that be correct? Any other insights or advice? I really need to get pytorch installed.

Output:
I recommend you create a new conda enviroment and try to reinstall PyTorch in this way:

To install PyTorch via Anaconda, and do not have a CUDA-capable[LINK] system or do not require CUDA, use the following conda command.

conda install pytorch-cpu torchvision-cpu -c pytorch

To install PyTorch via Anaconda, and you are using CUDA 9.0, use the following conda command:

conda install pytorch torchvision -c pytorch

CUDA 8.x

conda install pytorch torchvision cuda80 -c pytorch

CUDA 10.0

conda install pytorch torchvision cuda100 -c pytorch

#Inno


Instruction: No module named 'torch' or 'torch.C'

Input:
Would appreciate an explanation like I'm 5 simply because I have checked all relevant answers and none have helped.

I have installed Python.
I have installed Pycharm.
I have installed Anaconda.
I have installed Microsoft Visual Studio.
I have not installed the CUDA toolkit.

In Anaconda, I used the commands mentioned on Pytorch.org (06/05/18)

conda install pytorch -c pytorch
pip3 install torchvision

Both have downloaded and installed properly, and I can find them in my Users/Anaconda3/pkgs folder, which I have added to the Python path.

Trying to enter

import torch

in the Python console proved unfruitful - always giving me the same error,

  No module named 'torch'

I have also tried using the Project Interpreter to download the Pytorch package. It worked for numpy (sanity check, I suppose) but told me to go to Pytorch.org when I tried to install the "pytorch" or "torch" packages.

When trying to use the console in PyCharm, pip3 install codes (thinking maybe I need to save the packages into my current project, rather than in the Anaconda folder) return me an error message saying

  torch-0.4.0-cp35-cp35m-win_amd64.whl is not a supported wheel on this
  platform.

The same message shows no matter if I try downloading the CUDA version or not, or if I choose to use the 3.5 or 3.6 Python link (I have Python 3.7)

Currently the closest I have gotten to a solution, is manually copying the "torch" and "torch-0.4.0-py3.6.egg-info" folders into my current Project's lib folder.
However, when I do that and then run "import torch" I received the following error:

  Traceback (most recent call last):

  File "", line 1, in

  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.2\helpers\pydev_pydev_bundle\pydev_import_hook.py", line 19, in do_import

  module = self._system_import(name, *args, **kwargs)   File "C:\Users\Michael\PycharmProjects\Pytorch_2\venv\lib\site-packages\torch__init__.py",

  line 78, in

  from torch._C import *

  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.2\helpers\pydev_pydev_bundle\pydev_import_hook.py", line 19, in do_import

  module = self._system_import(name, *args, **kwargs) ModuleNotFoundError: No module named 'torch._C'

Output:
pip uninstall torch

then
pip install torch worked for me.


Instruction: PyTorch: Add validation error in training

Input:
I am using PyTorch to train a cnn model. Here is my Network architecture:

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as I

    class Net(nn.Module):

        def __init__(self):
            super(Net, self).__init__()

            self.conv1 = nn.Conv2d(1, 32, 5)
            self.pool = nn.MaxPool2d(2,2)
            self.conv1_bn = nn.BatchNorm2d(32)
            self.conv2 = nn.Conv2d(32, 64, 5)
            self.conv2_drop = nn.Dropout2d()
            self.conv2_bn = nn.BatchNorm2d(64)
            self.fc1 = torch.nn.Linear(53*53*64, 256)
            self.fc2 = nn.Linear(256, 136)

        def forward(self, x):

            x = F.relu(self.conv1_bn(self.pool(self.conv1(x))))
            x = F.relu(self.conv2_bn(self.pool(self.conv2_drop(self.conv2(x)))))
            x = x.view(-1, 53*53*64)
            x = F.relu(self.fc1(x))
            x = F.dropout(x, training=self.training)
            x = self.fc2(x)

            return x

Then I train the model like below:

# prepare the net for training
    net.train()

    for epoch in range(n_epochs):  # loop over the dataset multiple times

        running_loss = 0.0

        # train on batches of data, assumes you already have train_loader
        for batch_i, data in enumerate(train_loader):
            # get the input images and their corresponding labels
            images = data['image']
            key_pts = data['keypoints']

            # flatten pts
            key_pts = key_pts.view(key_pts.size(0), -1)

            # wrap them in a torch Variable
            images, key_pts = Variable(images), Variable(key_pts)

            # convert variables to floats for regression loss
            key_pts = key_pts.type(torch.FloatTensor)
            images = images.type(torch.FloatTensor)

            # forward pass to get outputs
            output_pts = net(images)

            # calculate the loss between predicted and target keypoints
            loss = criterion(output_pts, key_pts)

            # zero the parameter (weight) gradients
            optimizer.zero_grad()

            # backward pass to calculate the weight gradients
            loss.backward()

            # update the weights
            optimizer.step()

            # print loss statistics
            running_loss += loss.data[0]

I am wondering if it is possible to add the validation error in the training? I mean something like this (validation split) in Keras:

myModel.fit(trainX, trainY, epochs=50, batch_size=1, verbose=2, validation_split = 0.1)

Output:
Here is an example how to split your dataset for training and validation, then switch between the two phases every epoch:

import numpy as np
import torch
from torchvision import datasets
from torch.autograd import Variable
from torch.utils.data.sampler import SubsetRandomSampler

# Examples:
my_dataset = datasets.MNIST(root="/home/benjamin/datasets/mnist", train=True, download=True)
validation_split = 0.1

dataset_len = len(my_dataset)
indices = list(range(dataset_len))

# Randomly splitting indices:
val_len = int(np.floor(validation_split * dataset_len))
validation_idx = np.random.choice(indices, size=val_len, replace=False)
train_idx = list(set(indices) - set(validation_idx))

# Contiguous split
# train_idx, validation_idx = indices[split:], indices[:split]

## Defining the samplers for each phase based on the random indices:
train_sampler = SubsetRandomSampler(train_idx)
validation_sampler = SubsetRandomSampler(validation_idx)

train_loader = torch.utils.data.DataLoader(my_dataset, sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(my_dataset, sampler=validation_sampler)
data_loaders = {"train": train_loader, "val": validation_loader}
data_lengths = {"train": len(train_idx), "val": val_len}

# Training with Validation (your code + code from Pytorch tutorial: https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)
n_epochs = 40
net = ...

for epoch in range(n_epochs):
    print('Epoch {}/{}'.format(epoch, n_epochs - 1))
    print('-' * 10)

    # Each epoch has a training and validation phase
    for phase in ['train', 'val']:
        if phase == 'train':
            optimizer = scheduler(optimizer, epoch)
            net.train(True)  # Set model to training mode
        else:
            net.train(False)  # Set model to evaluate mode

        running_loss = 0.0

        # Iterate over data.
        for data in data_loaders[phase]:

            # get the input images and their corresponding labels
            images = data['image']
            key_pts = data['keypoints']

            # flatten pts
            key_pts = key_pts.view(key_pts.size(0), -1)

            # wrap them in a torch Variable
            images, key_pts = Variable(images), Variable(key_pts)

            # convert variables to floats for regression loss
            key_pts = key_pts.type(torch.FloatTensor)
            images = images.type(torch.FloatTensor)

            # forward pass to get outputs
            output_pts = net(images)

            # calculate the loss between predicted and target keypoints
            loss = criterion(output_pts, key_pts)

            # zero the parameter (weight) gradients
            optimizer.zero_grad()

            # backward + optimize only if in training phase
            if phase == 'train':
                loss.backward()
                # update the weights
                optimizer.step()

            # print loss statistics
            running_loss += loss.data[0]

        epoch_loss = running_loss / data_lengths[phase]
        print('{} Loss: {:.4f}'.format(phase, epoch_loss))


Instruction: Pytorch on Windows gives ImportError

Input:
I installed Pytorch for PYthon 3.6 using pip as instructed on https://pytorch.org/.

Pytorch is installed succesfully, but when I run code, I get this:

  File "C:\Users\\PycharmProjects\chatbot-light\pytorch\rnn_attention\seq2seq_translation_tutorial.py", line 93, in <module>
    import torch
  File "C:\Users\\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\__init__.py", line 78, in <module>
    from torch._C import *
ImportError: DLL load failed: The specified module could not be found

Output:
Pytorch devs recommend installing Pytorch using Anaconda.

Since Anaconda deals with all the dependencies you shouldn't have any DLL-related problems after installing Pytorch with it.


Instruction: RNN with many-to-one setup - which output to use

Input:
I going through a series of machine learning examples that use RNNs for document classification (many-to-one). In most tutorials, the RNN output of the last time step is used, i.e., fed into one or more dense layers to map it to the number of classes (e.g., [1], [2]).

However, I also came across some examples where, instead of the last output, the average of the outputs over all time steps is used (mean pooling?, e.g., [3]). The dimensions of this averaged output are of course the same as for the last output. So computationally, both approaches just work the same.

My questions is now, what is the intuition between the two different approaches. Due to recursive nature, the last output also reflects the output of the previous time steps. So why the idea of averaging the RNN outputs over all time steps. When to use what?

Output:
Pooling over time is a specific technique that is used to extract the features from the input sequence. From this question:

  The reason to do this, instead of "down-sampling" the sentence like in a CNN, is that in NLP the sentences naturally have different length in a corpus. This makes the feature maps different for different sentences, but we'd like to reduce the tensor to a fixed size to apply softmax or regression head in the end. As stated in the paper, it allows to capture the most important feature, one with the highest value for each feature map.

It's important to note here that max-over-time (or average-over-time) is usually an intermediate layer. In particular, there can be several of them in a row or in parallel (with different window size). The end result produced by the network can still be either many-to-one or many-to-many (at least in theory).

However, in most of the cases, there is a single output from the RNN. If the output must be a sequence, this output is usually fed into another RNN. So it all boils down to how exactly this single value is learned: take the last cell output or aggregate across the whole sequence or apply attention mechanism, etc.


Instruction: Pytorch: Softmax on each row of a 2d Tensor

Input: I need my neural net to output N distributions over A actions. each distribution should go through softmax. The sum of each row should then obviously be 1 and the sum of the whole layer should be N. Is there such functionality in PyTorch?

Output:
Use the softmax and specify the row as the dimension to operate on

import torch.nn.Functional as F

x = ...# your N x A input
x_distribution = F.softmax(x, dim = 1)


Instruction: PyTorch : error message "torch has no [...] member"

Input:
Good evening,
I have just installed PyTorch 0.4.0 and I'm trying to carry out the first tutorial "What is PyTorch?"
I have written a Tutorial.py file which I try to execute with Visual Studio Code

Here is the code :

from __future__ import print_function
import torch

print (torch.__version__)

x = x = torch.rand(5, 3)
print(x)

Unfortunately, when I try to debug it, i have an error message :
"torch has no rand member"

This is true with any member function of torch I may try

Can anybody help me please?

Output:
In case you haven't got a solution to your problem or someone else encounters it.

The error is raised because of Pylint (Python static code analysis tool) not recognizing rand as the member function. You can either configure Pylint to ignore this problem or you can whitelist torch (better solution) to remove lint errors by adding following to your .pylintrc file.

[TYPECHECK]

# List of members which are set dynamically and missed by Pylint inference
# system, and so shouldn't trigger E1101 when accessed.
generated-members=numpy.*, torch.*

In Visual Studio Code, you could also add the following to the user settings:

"python.linting.pylintArgs": [
"--generated-members=numpy.* ,torch.*"
]

The issue is discussed here on PyTorch GitHub page.


Instruction: Higher order gradients in pytorch

Input:
I have implemented the following Jacobian function in pytorch.  Unless I have made a mistake, it computes the Jacobian of any tensor w.r.t. any dimensional inputs:

import torch
import torch.autograd as ag

def nd_range(stop, dims = None):
    if dims == None:
        dims = len(stop)
    if not dims:
        yield ()
        return
    for outer in nd_range(stop, dims - 1):
        for inner in range(stop[dims - 1]):
            yield outer + (inner,)

def full_jacobian(f, wrt):
    f_shape = list(f.size())
    wrt_shape = list(wrt.size())
    fs = []

    f_range = nd_range(f_shape)
    wrt_range = nd_range(wrt_shape)

    for f_ind in f_range:
        grad = ag.grad(f[tuple(f_ind)], wrt, retain_graph=True, create_graph=True)[0]
        for i in range(len(f_shape)):
            grad = grad.unsqueeze(0)
        fs.append(grad)

    fj = torch.cat(fs, dim=0)
    fj = fj.view(f_shape + wrt_shape)
    return fj

On top of this, I have tried to implement a recursive function to calculate nth order derivatives:

def nth_derivative(f, wrt, n):
    if n == 1:
        return full_jacobian(f, wrt)
    else:
        deriv = nth_derivative(f, wrt, n-1)
        return full_jacobian(deriv, wrt)

I ran a simple test:

op = torch.ger(s, s)
deep_deriv = nth_derivative(op, s, 5)

Unfortunately, this succeeds in getting me the Hessian...but no higher order derivatives.  I'm aware many higher order derivatives should be 0, but I'd prefer if pytorch can analytically compute that.

One fix has been to change the gradient calculation to:

try:
            grad = ag.grad(f[tuple(f_ind)], wrt, retain_graph=True, create_graph=True)[0]
        except:
            grad = torch.zeros_like(wrt)

Is this the accepted correct way to handle this?  Or is there a better option?  Or do I have the reason for my issue completely wrong to begin with?

Output:
You can just iterate calling the grad function:

import torch
from torch.autograd import grad

def nth_derivative(f, wrt, n):

    for i in range(n):

        grads = grad(f, wrt, create_graph=True)[0]
        f = grads.sum()

    return grads

x = torch.arange(4, requires_grad=True).reshape(2, 2)
loss = (x ** 4).sum()

print(nth_derivative(f=loss, wrt=x, n=3))

outputs

tensor([[  0.,  24.],
        [ 48.,  72.]])


Instruction: CUDNN_STATUS_INTERNAL_ERROR when using both Pytorch and TensorFlow

Input:
I'm running a program to process some data, and I inference both a TensorFlow model and a Pytorch model.

When inferencing either of the models everything works fine. However, when I add the pytorch input my program crashes with this error:

2018-05-14 12:55:05.525251: E tensorflow/stream_executor/cuda/cuda_dnn.cc:385] could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR
2018-05-14 12:55:05.525280: F tensorflow/core/kernels/conv_ops.cc:717] Check failed: stream->parent()->GetConvolveAlgorithms( conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms)

Note that this already happens before I do anything with Pytorch. No models are loaded, nothing is put on GPU, no devices are checked.

Does anyone know what might be going wrong, how to fix it, and if there are some parameters I can change?

Something I already tried is disabling the PyTorch backend using this code:

import torch.backends.cudnn as cudnn
cudnn.enabled = False

But unfortunately this does not help...

Output:
You'll find in the NVIDIA Forums some references of cuBLAS not playing well with several Python processes interacting with it at the same time. This is referenced in this 1 year old issue for Tensorflow, but it should be the same for any multiple-PyTorch client applications interfacing with GPU through CUDA - and cuBLAS, to be more specific. cuBLAS handles weren't being properly initialized, somehow due to a mixture of issues related to on-disk caching and RAM utilization being too large.
The solution was both to delete the on-disk cache for cuBLAS,

sudo rm -rf ~/.nv

and restrict the amount of memory usage for nets.


Instruction: Pytorch autograd fails with "RuntimeError: differentiated input is unreachable" after collecting inputs

Input:
Pytorch version 0.3.1

EDIT: I am rewriting this question to be simpler, as I've narrowed down the bug.

I have some variables:

x = ag.Variable(torch.ones(1, 1), requires_grad = True)
y = ag.Variable(torch.ones(1, 1), requires_grad = True)
z = ag.Variable(torch.ones(1, 1), requires_grad = True)

I then create a variable representing their concatenation:

w = torch.cat([x, y, z])
f = x + y + z

Then I try to take derivatives:

ag.grad(f, x, retain_graph=True, create_graph=True)

This is fine and returns 1, as expected. Same for y and z.

However,

ag.grad(f, w, retain_graph=True, create_graph=True)

Returns an error: RuntimeError: differentiated input is unreachable

Of course that makes sense - w is not explicitly used in the declaration of f. However, I’d like a behavior where one line of code can generate something like [1; 1; 1] as output.

Let’s say I wanted to conveniently batch my variables together, and then take the gradient of the whole shebang at once, rather than processing variables independently (which can make bookkeeping a nightmare). Is there any way to get the outcome I desire?

Output:
Does something like this work or you want to keep f = x + y + z?

w = torch.cat([x, y, z])
f = w[0] + w[1] + w[2]
print (ag.grad(f, w, retain_graph=True, create_graph=True))

# output (tensor([[ 1.],[ 1.],[ 1.]]),)


Instruction: Why does my SRGAN (using PyTorch) result look similar to SRResNet results?

Input:
SRGAN was implemented using PyTorch.

Generator pre-train was conducted in 100 times and the SRGAN train was conducted in 200 times.

The code is a combination of the existing github codes.

For the content loss, MSELoss () in PyTorch was used and BCELoss () in PyTorch was used for adversarial loss.

When I ran code, LossD converges to 0, and LossG oscillates around a certain value. So I stopped training because I thought it was not training anymore.

If the training is 1e5 as in the paper, will the result change? Or is it a matter of loss function?

Below is the SRGAN training code.

print('Adversarial training')
for epoch in range(NUM_EPOCHS):
    train_bar = tqdm(train_loader)
    running_results = {'batch_sizes': 0, 'd_loss': 0, 'g_loss': 0, 'd_score': 0, 'g_score': 0}
    # train_bar = tqdm(train_loader)
    for data, target in train_bar:
        batch_size = data.size(0)
        running_results['batch_sizes'] += batch_size

        target_real = Variable(torch.ones(batch_size, 1))
        target_fake = Variable(torch.zeros(batch_size, 1))

        if torch.cuda.is_available():
            target_real = target_real.cuda()
            target_fake = target_fake.cuda()

        real_img = Variable(target)
        z = Variable(data)

        # Generate real and fake inputs
        if torch.cuda.is_available():
            inputsD_real = real_img.cuda()
            inputsD_fake = netG(z.cuda())
        else:
            inputsD_real = real_img
            inputsD_fake = netG(z)

        ######### Train discriminator #########
        netD.zero_grad()

        # With real data
        outputs = netD(inputsD_real)
        D_real = outputs.data.mean()

        lossD_real = adversarial_criterion(outputs, target_real)

        # With fake data
        outputs = netD(inputsD_fake.detach()) # Don't need to compute gradients wrt weights of netG (for efficiency)
        D_fake = outputs.data.mean()

        lossD_fake = adversarial_criterion(outputs, target_fake)

        lossD_total = lossD_real + lossD_fake

        lossD_total.backward()

        # Update discriminator weights
        optimizerD.step()

        ######### Train generator #########
        netG.zero_grad()

        real_features = Variable(feature_extractor(inputsD_real).data)
        fake_features = feature_extractor(inputsD_fake)

        lossG_vgg19 = content_criterion(fake_features, real_features)
        lossG_adversarial = adversarial_criterion(netD(inputsD_fake).detach(), target_real)
        lossG_mse = content_criterion(inputsD_fake, inputsD_real)

        lossG_total = lossG_mse + 2e-6 * lossG_vgg19 + 0.001 * lossG_adversarial
        lossG_total.backward()

        # Update generator weights
        optimizerG.step()

Output: If you use the classic GAN model for alternate training, this situation is inevitable, try to change the training method


Instruction: PyTorch: DecoderRNN: RuntimeError: input must have 3 dimensions, got 2

Input:
I am building a DecoderRNN using PyTorch (This is an image-caption decoder):

class DecoderRNN(nn.Module):
    def __init__(self, embed_size, hidden_size, vocab_size):

        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size
        self.gru = nn.GRU(embed_size, hidden_size, hidden_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, features, captions):

        print (features.shape)
        print (captions.shape)
        output, hidden = self.gru(features, captions)
        output = self.softmax(self.out(output[0]))
        return output, hidden

The data have the following shapes:

torch.Size([10, 200])  <- features.shape (10 for batch size)
torch.Size([10, 12])   <- captions.shape (10 for batch size)

Then I got the following errors. Any ideas what I missed here? Thanks!

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-2-76e05ba08b1d> in <module>()
     44         # Pass the inputs through the CNN-RNN model.
     45         features = encoder(images)
---> 46         outputs = decoder(features, captions)
     47
     48         # Calculate the batch loss.

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/home/workspace/model.py in forward(self, features, captions)
     37         print (captions.shape)
     38         # features = features.unsqueeze(1)
---> 39         output, hidden = self.gru(features, captions)
     40         output = self.softmax(self.out(output[0]))
     41         return output, hidden

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    167             flat_weight=flat_weight
    168         )
--> 169         output, hidden = func(input, self.all_weights, hx)
    170         if is_packed:
    171             output = PackedSequence(output, batch_sizes)

/opt/conda/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward(input, *fargs, **fkwargs)
    383             return hack_onnx_rnn((input,) + fargs, output, args, kwargs)
    384         else:
--> 385             return func(input, *fargs, **fkwargs)
    386
    387     return forward

/opt/conda/lib/python3.6/site-packages/torch/autograd/function.py in _do_forward(self, *input)
    326         self._nested_input = input
    327         flat_input = tuple(_iter_variables(input))
--> 328         flat_output = super(NestedIOFunction, self)._do_forward(*flat_input)
    329         nested_output = self._nested_output
    330         nested_variables = _unflatten(flat_output, self._nested_output)

/opt/conda/lib/python3.6/site-packages/torch/autograd/function.py in forward(self, *args)
    348     def forward(self, *args):
    349         nested_tensors = _map_variable_tensor(self._nested_input)
--> 350         result = self.forward_extended(*nested_tensors)
    351         del self._nested_input
    352         self._nested_output = result

/opt/conda/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward_extended(self, input, weight, hx)
    292             hy = tuple(h.new() for h in hx)
    293
--> 294         cudnn.rnn.forward(self, input, hx, weight, output, hy)
    295
    296         self.save_for_backward(input, hx, weight, output)

/opt/conda/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in forward(fn, input, hx, weight, output, hy)
    206         if (not is_input_packed and input.dim() != 3) or (is_input_packed and input.dim() != 2):
    207             raise RuntimeError(
--> 208                 'input must have 3 dimensions, got {}'.format(input.dim()))
    209         if fn.input_size != input.size(-1):
    210             raise RuntimeError('input.size(-1) must be equal to input_size. Expected {}, got {}'.format(

RuntimeError: input must have 3 dimensions, got 2

Output:
Your GRU input needs to be 3-dimensional:

  input of shape (seq_len, batch, input_size): tensor containing the features of the input sequence.

Further you need to provide the hidden state (last encoder hidden state in this case) as second parameter:

self.gru(input, h_0)

Where input is your actual input and h_0 the hidden state which needs to be 3-dimensional as well:

  h_0 of shape (num_layers * num_directions, batch, hidden_size): tensor
  containing the initial hidden state for each element in the batch.
  Defaults to zero if not provided.

https://pytorch.org/docs/master/nn.html#torch.nn.GRU


Instruction: How to build an autograd-compatible Pytorch module that resizes tensors like images?

Input:
I was wondering if I can build an image resize module in Pytorch that takes a torch.tensor of 3*H*W as the input and return a tensor as the resized image.

I know it is possible to convert tensor to PIL Image and use torchvision,
but I also hope to back propagate gradients from the resized image to the original image, and the following example will return such error (in PyTorch 0.4.0 on Windows 10):

import numpy as np
from torchvision import transforms

t2i = transforms.ToPILImage()
i2t = transforms.ToTensor()

trans = transforms.Compose(
    t2i, transforms.Resize(size=200), i2t]
)

test = np.random.normal(size=[3, 300, 300])
test = torch.tensor(test, requires_grad=True)
resized = trans(test)
resized.backward()

print(test.grad)

Traceback (most recent call last):
  File "D:/Projects/Python/PyTorch/test.py", line 41, in <module>
    main()
  File "D:/Projects/Python/PyTorch/test.py", line 33, in main
    resized = trans(test)
  File "D:\Anaconda3\envs\pytorch\lib\site-packages\torchvision\transforms\transforms.py", line 42, in __call__
    img = t(img)
  File "D:\Anaconda3\envs\pytorch\lib\site-packages\torchvision\transforms\transforms.py", line 103, in __call__
    return F.to_pil_image(pic, self.mode)
  File "D:\Anaconda3\envs\pytorch\lib\site-packages\torchvision\transforms\functional.py", line 102, in to_pil_image
    npimg = np.transpose(pic.numpy(), (1, 2, 0))
RuntimeError: Can't call numpy() on Variable that requires grad. Use var.detach().numpy() instead.

It seems like I cannot "imresize" a tensor without detaching it from autograd first, but detaching it prevents me from computing gradients.

Is there a way to build a torch function/module that does the same thing as torchvision.transforms.Resize that is autograd compatiable? Any help is much appreciated!

Output: torch.nn.functional.upsample works for me, ypa!


Instruction: How do I load up an image and convert it to a proper tensor for PyTorch?

Input:
I'm trying to custom load some image files (JPG files) with some labels and feed them into a convolutional neural network (CNN) in PyTorch following the example here. However, there still seem to be no decent end-to-end tutorials. The problem that I am seeing is the following.

RuntimeError: thnn_conv2d_forward is not implemented for type
torch.ByteTensor

My Dataset looks like the following.

class ImageData(Dataset):
    def __init__(self, width=256, height=256, transform=None):
        self.width = width
        self.height = height
        self.transform = transform
        y, x = get_images() #y is a list of labels, x is a list of file paths
        self.y = y
        self.x = x

    def __getitem__(self, index):
        img = Image.open(self.x[index]) # use pillow to open a file
        img = img.resize((self.width, self.height)) # resize the file to 256x256
        img = img.convert('RGB') #convert image to RGB channel
        if self.transform is not None:
            img = self.transform(img)

        img = np.asarray(img).transpose(-1, 0, 1) # we have to change the dimensions from width x height x channel (WHC) to channel x width x height (CWH)
        img = torch.from_numpy(np.asarray(img)) # create the image tensor
        label = torch.from_numpy(np.asarray(self.y[index]).reshape([1, 1])) # create the label tensor
        return img, label

    def __len__(self):
        return len(self.x)

The CNN is taken from here and is modified to handle NCWH (batch x channel x width x height) as follows.

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 256, 256)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

The learning loop is also taken from the same tutorial and looks like the following.

for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(dataloader, 0):
        # get the inputs
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')

However, the RuntimeError mentioned above is thrown. Any ideas on what I am doing wrong?

Additionally, I know that without transposing the image data, it is in the shape of WHC, but the NN model requires it as CWH. The problem is this, if we change from WHC to CWH, then we can no longer simply plot the images if we iterate over the DataLoader.

data = ImageData()
dataloader = DataLoader(data, batch_size=10, shuffle=True, num_workers=1)
imgs, labels = next(iter(dataloader))
plt.imshow(imgs.numpy()[0,:,:,:])
plt.show()

Attempting to do will throw the following error.

TypeError: Invalid dimensions for image data

To me, that Pillow gives you WHC and you can use that to plot, but the PyTorch CNN wants CWH to process, is a nuisance. Any idea on how to consistently or easily not do so many transforms but be able to plot and feed the data into the CNN? Or is this mismatch of WHC vs CWH just something we have to live with?

Without transposing the image, when feeding it to the CNN, the following error is thrown.

RuntimeError: Given groups=1, weight[256, 3, 256, 256], so expected
input[10, 256, 256, 3] to have 3 channels, but got 256 channels

instead.

Output:
conv2d operates on float tensors. It's common and good practice to normalize input images before passing them into the neural network.

I would add the line img = img/255 immediately before you convert it to a Torch tensor in __getitem__, then it will be converted to a float tensor rather than a byte tensor and thus will be compatible with the conv2d method.


Instruction: How to Train Word Embedding With Pytorch

Input:
I am trying learn and practice how to train the embedding of Vocabulary set using pytorch.

https://pytorch.org/tutorials/beginner/nlp/word_embeddings_tutorial.html

loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))

At the above tutorial example, the loss was calculated between log_probs which is 4 x 10 tensor (4 for context words number and 10 is embedding_dimension) and the word index of target which is integer ranged from 0 to 49.

I can't understand why the code doesn't compare between its context embedding with its target embedding but just compare with its class index, which is a mere integer holding no information..

I think one must go back to the embedding parameter then call it, and compare it with its context I guess.

Is it just because it's only for the tutorial or I am misinterpreting some point?

Thanks for your help in advance.

Output: The embedding is a by-product of training your model. The model itself is trained with supervised learning to predict the next word give the context words. This is usually done (also in that tutorial) in the form of a one-hot encoder. The log_probs are the output of the model that are probabilities in logarithmic form which is then compared to one-hot encoder target. Higher probabilities for the correct target word corresponds to lower loss and inversely, lower probabilities for correct target words will cause larger loss signals to propagate through the network and change the weights. That is why the log_probs are compared against the target.


Instruction: Importing PyTorch in PyCharm using Anaconda

Input: I just installed PyCharm and Anaconda. I installed PyTorch to Anaconda and i can even use "import torch" in Anaconda. I've created a new Project in PyCharm with the Anaconda Interpreter but i still can't use PyTorch in PyCharm.

Output:
I had this problem also. Program that imported torch worked fine at anaconda prompt running in my pytorch env,  but when i ran pycharm from the windows shortcut and EVEN set my environment to use pytorch env, it would complain torch could not be imported.
When i ran pycharm from the prompt as shown, it worked.


Instruction: ‘DataParallel’ object has no attribute ‘init_hidden’

Input:
What I want to do is using DataParallel in my custom RNN class.
It seems like I initialized hidden_0 in a wrong way...

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, n_layers=1):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers

        self.encoder = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, n_layers,batch_first = True)
        self.decoder = nn.Linear(hidden_size, output_size)
        self.init_hidden(batch_size)

    def forward(self, input):
        input = self.encoder(input)
        output, self.hidden = self.gru(input,self.hidden)
        output = self.decoder(output.contiguous().view(-1,self.hidden_size))
        output = output.contiguous().view(batch_size,num_steps,N_CHARACTERS)
        #print (output.size())10,50,67

        return output

    def init_hidden(self,batch_size):
        self.hidden = Variable(T.zeros(self.n_layers, batch_size, self.hidden_size).cuda())

And I call the network in this way:
decoder = T.nn.DataParallel(RNN(N_CHARACTERS, HIDDEN_SIZE, N_CHARACTERS), dim=1).cuda()

Then start training:
for epoch in range(EPOCH_):
    hidden = decoder.init_hidden()

But I get the error and I have no ideal how to fix it…

'DataParallel' object has no attribute 'init_hidden'

Thanks for your help!

Output:
When using DataParallel your original module will be in attribute module of the parallel module:

for epoch in range(EPOCH_):
    hidden = decoder.module.init_hidden()


Instruction: RuntimeError: addmm(): argument 'mat1' (position 1) must be Variable, not torch.FloatTensor

Input:
I am running this extremely simple PyTorch example NN from the documentation as is, with nothing at all changed.

I get this error:

Traceback (most recent call last):
  File "<stdin>", line 2, in <module>
  File "/opt/conda/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "/opt/conda/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/container.py", line 67, in forward
    input = module(input)
  File "/opt/conda/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "/opt/conda/envs/fastai/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 55, in forward
    return F.linear(input, self.weight, self.bias)
  File "/opt/conda/envs/fastai/lib/python3.6/site-packages/torch/nn/functional.py", line 835, in linear
    return torch.addmm(bias, input, weight.t())
RuntimeError: addmm(): argument 'mat1' (position 1) must be Variable, not torch.FloatTensor

Apparently during the matrix multiplication, there is some data type error.
Why would the matrices I'm trying to multiply need to be Variable anyway?

I can do
x = Variable(torch.randn(N, D_in))
y = Variable(torch.randn(N, D_out))
but get
AttributeError: 'Variable' object has no attribute 'item'
so that didn't help.

I am running PyTorch version 0.3.1.post2.

Output:
I had the same issue, so I will just add the update commands in case you don't want to search them:
Optional:

conda list | grep pytorch
conda upgrade conda

The following update didn't update anything, although they should and I think its the right way to update (you may try it first)

conda update pytorch torchvision

What did help was to specify the version explicitly:

conda install pytorch=0.4.0 -c pytorch


Instruction: Issue while reshaping a torch tensor from [10,200,1] to [2000,1,1]

Input:
I am having a problem when trying to reshape a torch tensor Yp of dimensions [10,200,1] to [2000,1,1]. The tensor is obtained from a numpy array y of dimension [2000,1]. I am doing the following:

Yp = reshape(Yp, (-1,1,1))

I try to subtract the result to a torch tensor version of y by doing:

Yp[0:2000,0] - torch.from_numpy(y[0:2000,0])

I expect the result to be an array of zeros, but that is not the case. Calling different orders when reshaping (order = 'F' or 'C') does not solve the problem, and strangely outputs the same result when doing the subtraction. I only manage to get an array of zeros by calling on the tensor Yp the ravel method with order = 'F'.

What am I doing wrong? I would like to solve this using reshape!

Output:
I concur with @linamnt's comment (though the actual resulting shape is [2000, 1, 2000]).

Here is a small demonstration:

import torch
import numpy as np

# Your inputs according to question:
y = np.random.rand(2000, 1)
y = torch.from_numpy(y[0:2000,0])
Yp = torch.reshape(y, (10,200,1))

# Your reshaping according to question:
Yp = torch.reshape(Yp, (-1,1,1))
# (note: Tensor.view() may suit your need more if you don't want to copy values)

# Your subtraction:
y_diff = Yp - y
print(y_diff.shape)
# > torch.Size([2000, 1, 2000])
# As explained by @linamnt, unwanted broadcasting is done
# since the dims of your tensors don't match

# If you give both your tensors the same shape, e.g. [2000, 1, 1] (or [2000]):
y_diff = Yp - y.view(-1, 1, 1)
print(y_diff.shape)
# > torch.Size([2000, 1, 1])

# Checking the result tensor contains only 0 (by calculing its abs. sum):
print(y_diff.abs().sum())
# > 0


Instruction: Pytorch - Incorrect dimensions when using LSTM network

Input:
I'm getting started with pytorch and used a few transformation to build the following model using one of the tutorials as a reference:

model = torch.nn.Sequential(
     torch.nn.Linear(D_in, H),
     torch.nn.ReLU(),
     torch.nn.Linear(H, D_out),
)

I want to use an LSTM network, so I tried to do the following:

model = torch.nn.Sequential(
      torch.nn.LSTM(D_in, H),
      torch.nn.Linear(H, D_out)
)

which gives me this error:

  RuntimeError: input must have 3 dimensions, got 2

Why am I seeing this error? I anticipate there's something fundamentally wrong in my understanding of how transformations (networks?) can be chained in pytorch...

EDIT

After following @esBee's suggestion, I found that following runs correctly. This is because an LSTM expects the input to be of the folowing dimension:

  input of shape (seq_len, batch, input_size): tensor containing the features of the input sequence. The input can also be a packed variable length sequence

local_x = local_x.unsqueeze(0)
y_pred, (hn, cn) = layerA(local_x)
y_pred = y_pred.squeeze(0)
y_pred = layerB(y_pred)

However, the fact that my original training/test dataset is only of sequence length 1 makes me feel like I'm doing something incorrectly. What is the purpose of this parameter in the context of neural networks?

Output:
The thing you have to pay attention to here, is that, as opposed to linear layers such as torch.nn.Linear, there's more than 1 output for recurring layers such as torch.nn.LSTM.

While torch.nn.Linear returns simply the y in y = Ax + b, torch.nn.LSTMs return output, (h_n, c_n) (explained in more detail in the docs) to let you choose which output you want to process. So what happens in your example is you are feeding all these several types of output into the layer after your LSTM layer (leading to the error you are seeing). You should instead choose a specific part of the output of your LSTM and feed only that into the next layer.

Sadly I don't know how to choose the output of an LSTM within a Sequential (suggestions welcome), but you can rewrite

model = torch.nn.Sequential(
    torch.nn.LSTM(D_in, H),
    torch.nn.Linear(H, D_out)
)

model(x)

as

layerA = torch.nn.LSTM(D_in, H)
layerB = torch.nn.Linear(H, D_out)

x = layerA(x)
x = layerB(x)

and then correct it by choosing the output features (h_n) of the last layer of your LSTM by writing

layerA = torch.nn.LSTM(D_in, H)
layerB = torch.nn.Linear(H, D_out)

x = layerA(x)[0]
x = layerB(x)


Instruction: How do I calculate the confusion matrix in PyTorch efficiently?

Input: I have a tensor that contains my predictions and a tensor that contains the actual labels for my binary classification problem. How can I calculate the confusion matrix efficiently?

Output:
After my first version using a for-loop has proven inefficient, this is the fastest solution I came up with so far, for two equal-dimensional tensors prediction and truth:

def confusion(prediction, truth):
    confusion_vector = prediction / truth

    true_positives = torch.sum(confusion_vector == 1).item()
    false_positives = torch.sum(confusion_vector == float('inf')).item()
    true_negatives = torch.sum(torch.isnan(confusion_vector)).item()
    false_negatives = torch.sum(confusion_vector == 0).item()

    return true_positives, false_positives, true_negatives, false_negatives

Commented version and test-case at https://gist.github.com/the-bass/cae9f3976866776dea17a5049013258d


Instruction: from torch._C import * ImportError: DLL load failed: The specified module could not be found

Input:
I am trying to import torch in my windows machine using python 3.5. (CPU only)(pip)
I have followed the steps given in the official website.

When I try to import torch it gives me the error:

from torch._C import *
ImportError: DLL load failed: The specified module could not be found.

I have looked up from torch._C import * (ImportError: DLL load failed: The specified module could not be found. and ModuleNotFoundError: No module named 'torch._C' and followed their instructions, still the error persists.

The total traceback:

Traceback (most recent call last):
File "D:\CFPS.py", line 1, in <module>
import torch
File "C:\Program Files (x86)\Python35\lib\site-packages\torch\__init__.py",
line 78, in <module>
from torch._C import *
ImportError: DLL load failed: The specified module could not be found.
[Finished in 0.6s]

Is there a way to get past through this?

Output:
I've been encountering the same problem.   Pytorch seems to require openmp, however this is not part of the PIP distribution.

If you install Pytorch through Anaconda, the Anaconda install includes openmp, so this problem goes away.

To resolve this problem with pip, you can
pip install intel-openmp

but you still have to copy the openmp binaries from Python\Library\bin over to
Python\Lib\site-packages\torch\lib

Here's a link to the pytorch issue

pytorch issue


Instruction: LBFGS never converges in large dimensions in pytorch

Input:
I am playing with Rule 110 of Wolfram cellular automata. Given line of zeroes and ones, you can calculate next line with these rules:

Starting with 00000000....1 in the end you get this sequence:

Just of curiosity I decided to approximate these rules with a polynomial, so that cells could be not only 0 and 1, but also gray color in between:

def triangle(x,y,z,v0):
    v=(y + y * y + y * y * y - 3. * (1. + x) * y * z + z * (1. + z + z * z)) / 3.
    return (v-v0)*(v-v0)

so if x,y,z and v0 matches any of these rules from the table, it will return 0, and positive nonzero value otherwise.

Next I've added all possible groups of 4 neighbors into single sum, which will be zero for integer solutions:

def eval():
    s = 0.
    for i in range(W - 1):
        for j in range(1, W + 1):
            xx = x[i, (j - 1) % W]
            yy = x[i, j % W]
            zz = x[i, (j + 1) % W]
            r = x[i + 1, j % W]
            s += triangle(xx, yy, zz, r)
    for j in range(W - 1): s += x[0, j] * x[0, j]
    s += (1 - x[0, W - 1]) * (1 - x[0, W - 1])
    return torch.sqrt(s)

Also in the bottom of this function I add ordinary conditions for first line, so that all elements are 0 except last one, which is 1. Finally I've decided to minimize this sum of squares on W*W matrix with pytorch:

x = Variable(torch.DoubleTensor(W,W).zero_(), requires_grad=True)
opt = torch.optim.LBFGS([x],lr=.1)
for i in range(15500):
    def closure():
        opt.zero_grad()
        s=eval()
        s.backward()
        return s
    opt.step(closure)

Here is full code, you can try it yourself. The problem is that for 10*10 it converges to correct solution in ~20 steps:

But if I take 15*15 board, it never finishes convergence:

The graph on the right shows how sum of squares is changing with each next iteration and you can see that it never reaches zero. My question is why this happens and how can I fix this. Tried different pytorch optimisers, but all of them perform worse then LBFGS. Tried different learning rates. Any ideas why this happen and how I can reach final point during optimisation?

UPD: improved convergence graph, log of SOS:

UPD2: I also tried doing same in C++ with dlib, and I don't have any convergency issues there, it goes much deeper in much less time:

I am using this code for optimisation in C++:

find_min_using_approximate_derivatives(bfgs_search_strategy(),
        objective_delta_stop_strategy(1e-87),
        s, x, -1)

Output:
What's you're trying to do here is non convex optimisation and this is a notoriously difficult problem. Once you think about it, it make sense because just about any practical mathematical problem can be formulated as an optimisation problem.

1. Prelude
So, before giving you hints as to where to find a solution to your particular problem, I want to illustrate why certain optimisation problems are easy to solve.

I'm going to start by discussing convex problems. These are easy to solve even in the constrained case, and the reason for this is that when you compute the gradient you actually get a lot of information of where the minimum cannot be (the Taylor expansion of a convex function, f, is always an underestimate of f), additionally there is only one minimum and no sadle points. If you're interested in learning more about convex optimisation I recommend seeing Stephen Boyd's class in convex optimisation on YouTube

Now then if non convex optimization is so difficult, how come we are able to solve it in deep learning? The answer is simply that the non convex function we are minimising in deep learning it's quite nice as demonstrated by Henaff et al.

It is therefore important that machine learning practitioners realise that the operation procedures used in deep learning will most likely not yield a good minimum, if they converge to a minimum in the first place, on other non-convex problems.

2. Answer to your question
Now then to answer your problem, You're not you probably not gonna find and fast solution as nonconvex optimisation is NP complete. But fear not, SciPy has a few global optimisation algorithms to choose from. Here is a link to another stack overflow thread with a good answer to your question.

3. Moral of the story
Finally, I want to remind you that convergence guarantees are important, forgetting it has led to an oil rig collapsing.

PS. Please forgive typos, I'm usong my phone for this

Update: As to why BFGS works with dlib, there might be two reasons, firstly, BFGS is better at using curvature information than L-BFGS, and secondly it uses a line search to find an optimal step size. I'd recommend checking if PyTorch allow line searches and if not, setting an decreasing step size (or just a really low one).


Instruction: How can I enable pytorch GPU support in Google Colab?

Input:
How can I enable pytorch to work on GPU?

I've installed pytorch successfully in google colab notebook:

Tensorflow reports GPU to be in place:

But torch.device function fails somehow:

How can I fix this?

Output:
I hit the same issue.

Try installing Torch like this:

# http://pytorch.org/
from os import path
from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag
platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())

accelerator = 'cu80' #'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'
print('Platform:', platform, 'Accelerator:', accelerator)

!pip install --upgrade --force-reinstall -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.0-{platform}-linux_x86_64.whl torchvision

import torch
print('Torch', torch.__version__, 'CUDA', torch.version.cuda)
print('Device:', torch.device('cuda:0'))

The output should be:

  Platform: cp36-cp36m Accelerator: cu80 Torch 0.4.0 CUDA 8.0.61
  Device: cuda:0

Some snippets floating around use torch-0.3.0.post4-{platform}-linux_x86_64.whl, which will lead to the same error, because device is a Torch 4 feature.  If you have already installed the wrong version, you may need to do !pip uninstall torch.

Also be sure to enable GPU under Edit > Notebook settings > Hardware accelerator.


Instruction: PyTorch - "Attribute Error: module 'torch' has no attribute 'float'

Input:
I'm trying to implement the Spatial Transformer Network from here and I am running into this issue:

class STNLayer(torch.nn.Module):
    def __init__(self, input_size):
        super(STNLayer, self).__init__()
        self.input_size = input_size
        self.localization = nn.Sequential(
            nn.Conv2d(self.input_size, 8, kernel_size = 7),
            nn.MaxPool2d(2, stride = 2),
            nn.ReLU(True),
            nn.Conv2d(8, 10, kernel_size = 5),
            nn.MaxPool2d(2, stride = 2),
            nn.ReLU(True)
        )

        self.fc_loc = nn.Sequential(
            nn.Linear(10 * 12 * 12, 32),
            nn.ReLU(True),
            #nn.BatchNorm1d(32),
            nn.Linear(32, 3*2)
        )

        # Initialize weights to identity transformation
        self.fc_loc[2].weight.data.zero_()
        self.fc_loc[2].bias.data = torch.cuda.FloatTensor([1,0,0,0,1,0])

The line

self.fc_loc[2].bias.data = torch.cuda.FloatTensor([1,0,0,0,1,0])

is giving the error:

*** AttributeError: module 'torch' has no attribute 'float'

How do I fix this?

Output:
About the AttributeError

AttributeError: module 'torch' has no attribute 'float'

This AttributeError implies that somewhere in the code must be something like torch.float.

In your code example I cannot find anything like it.
However, the link you referenced for the code contains the following line:

self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))

PyTorch data types like torch.float came with PyTorch 0.4.0, so when you use something like torch.float in earlier versions like 0.3.1 you will see this error, because torch then actually has no attribute float.

If you have a line like in the example you've linked, it makes perfectly sense to get an error like this. For the code you've posted it makes no sense.

As you did not include a full error traceback I can only conjecture what the problem is. So probably you either have somewhere used torch.float in your code or you have imported some code with torch.float.
So what can you do to fix it?

Easiest way would be just updating PyTorch to 0.4.0 or higher.

If you don't want to update or if you are not able to do so for some reason. You just need to find the line (or lines) where torch.float is used and change it.

So probably something like this:

torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float)

and change it to:

torch.FloatTensor([1,0,0,0,1,0])

Similarly to the line you posted in your question.

I hope this helps!

Short Add-on for Jupyter Notebook:

This is just a side node, because your code and error message do not match: When importing code to Jupyter Notebook it is safest to restart the kernel after doing changes to the imported code. Otherwise already loaded modules are omitted during import and changes are not applied. So if there was an error in the old code this error might still occur and the traceback then points to the line you have just corrected.

So for example when changing in the imported code: torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float) to torch.FloatTensor([1,0,0,0,1,0]) it might still complain about torch.float even if the line then doesn't contain a torch.floatanymore (it even shows the new code in the traceback). This is kind of confusing because the traceback then shows an error which doesn't make sense for the given line.

In such a case restarting the kernel helps.


Instruction: Anchor boxes and offsets in SSD object detection

Input:
How do you calculate anchor box offsets for object detection in SSD? As far as I understood anchor boxes are the boxes in 8x8 feature map, 4x4 feature map, or any other feature map in the output layer.

So what are the offsets?

Is it the  distance between the centre of the bounding box and the centre of a particular box in say a 4x4 feature map?

If I am using a 4x4 feature map as my output, then my output should be of the dimension:

(4x4, n_classes + 4)

where 4 is for my anchor box co-ordinates.
This 4 co-ordinates can be something like:

(xmin, xmax, ymin, ymax)

This will correspond to the top-left and bottom-right corners of the bounding box.
So why do we need offsets and if so how do we calculate them?

Any help would be really appreciated!

Output:
We need offsets because thats what we calculate when we default anchor boxes, In case of ssd for every feature map cell they will have predefined number of anchor boxes of different scale ratios on very feature map cell,I think in the paper this number is 6.

Now because this is a detection problem ,we will also have ground truth bounding boxes,Here roughly, we compare the IOU of the anchor box to the GT box and if it is greater than a threshold say 0.5 we predict the box offsets to that anchor box.


Instruction: Column/row slicing a torch sparse tensor

Input:
I have a pytorch sparse tensor that I need sliced row/column wise using this slice [idx][:,idx] where idx is a list of indexes, using the mentioned slice yields my desired result on an ordinary float tensor. Is it possible applying the same slicing on a sparse tensor? Example here:

#constructing sparse matrix
i = np.array([[0,1,2,2],[0,1,2,1]])
v = np.ones(4)
i = torch.from_numpy(i.astype("int64"))
v = torch.from_numpy(v.astype("float32"))
test1 = torch.sparse.FloatTensor(i, v)

#constructing float tensor
test2 = np.array([[1,0,0],[0,1,0],[0,1,1]])
test2 = autograd.Variable(torch.cuda.FloatTensor(test2), requires_grad=False)

#slicing
idx = [1,2]
print(test2[idx][:,idx])

output:

Variable containing:
 1  0
 1  1
[torch.cuda.FloatTensor of size 2x2 (GPU 0)]

I am holding a 250.000 x 250.000 adjacency matrix, where I need to slice n rows and n columns, using the random idx, by simply sampling n random idx's. Since the dataset is so large it is not realistic to convert to a more convenient datatype.

can I achieve the same slicing result on test1? Is it even possible? If not, are there any work-arounds?

Right now I am running my model with the following "hack" of a solution:

idx = sorted(random.sample(range(0, np.shape(test1)[0]), 9000))
test1 = test1AsCsr[idx][:,idx].todense().astype("int32")
test1 = autograd.Variable(torch.cuda.FloatTensor(test1), requires_grad=False)

Where test1AsCsr is my test1 converted to a numpy CSR matrix. This solution works, it is however very slow, and makes my GPU utilization very low, since it needs to read/write from CPU memory, constantly.

Edit: Its fine with a non-sparse tensor as result

Output:
Possible answer for 2-dimentional sparse indices

Find an answer below, playing with several pytorch methods (torch.eq(), torch.unique(), torch.sort(), etc.) in order to output a compact, sliced tensor of shape (len(idx), len(idx)).

I tested several edge cases (unordered idx, v with 0s, i with multiple same index pairs, etc.), though I may have forgot some. Performance should also be checked.

import torch
import numpy as np

def in1D(x, labels):
    """
    Sub-optimal equivalent to numpy.in1D().
    Hopefully this feature will be properly covered soon
    c.f. https://github.com/pytorch/pytorch/issues/3025
    Snippet by Aron Barreira Bordin
    Args:
        x (Tensor):             Tensor to search values in
        labels (Tensor/list):   1D array of values to search for

    Returns:
        Tensor: Boolean tensor y of same shape as x, with y[ind] = True if x[ind] in labels

    Example:
        >>> in1D(torch.FloatTensor([1, 2, 0, 3]), [2, 3])
        FloatTensor([False, True, False, True])
    """
    mapping = torch.zeros(x.size()).byte()
    for label in labels:
        mapping = mapping | x.eq(label)
    return mapping

def compact1D(x):
    """
    "Compact" values 1D uint tensor, so that all values are in [0, max(unique(x))].
    Args:
        x (Tensor): uint Tensor

    Returns:
        Tensor: uint Tensor of same shape as x

    Example:
        >>> densify1D(torch.ByteTensor([5, 8, 7, 3, 8, 42]))
        ByteTensor([1, 3, 2, 0, 3, 4])
    """
    x_sorted, x_sorted_ind = torch.sort(x, descending=True)
    x_sorted_unique, x_sorted_unique_ind = torch.unique(x_sorted, return_inverse=True)
    x[x_sorted_ind] = x_sorted_unique_ind
    return x

# Input sparse tensor:
i = torch.from_numpy(np.array([[0,1,4,3,2,1],[0,1,3,1,4,1]]).astype("int64"))
v = torch.from_numpy(np.arange(1, 7).astype("float32"))
test1 = torch.sparse.FloatTensor(i, v)
print(test1.to_dense())
# tensor([[ 1.,  0.,  0.,  0.,  0.],
#         [ 0.,  8.,  0.,  0.,  0.],
#         [ 0.,  0.,  0.,  0.,  5.],
#         [ 0.,  4.,  0.,  0.,  0.],
#         [ 0.,  0.,  0.,  3.,  0.]])

# note: test1[1, 1] = v[i[1,:]] + v[i[6,:]] = 2 + 6 = 8
#       since both i[1,:] and i[6,:] are [1,1]

# Input slicing indices:
idx = [4,1,3]

# Getting the elements in `i` which correspond to `idx`:
v_idx = in1D(i, idx).byte()
v_idx = v_idx.sum(dim=0).squeeze() == i.size(0) # or `v_idx.all(dim=1)` for pytorch 0.5+
v_idx = v_idx.nonzero().squeeze()

# Slicing `v` and `i` accordingly:
v_sliced = v[v_idx]
i_sliced = i.index_select(dim=1, index=v_idx)

# Building sparse result tensor:
i_sliced[0] = compact1D(i_sliced[0])
i_sliced[1] = compact1D(i_sliced[1])

# To make sure to have a square dense representation:
size_sliced = torch.Size([len(idx), len(idx)])
res = torch.sparse.FloatTensor(i_sliced, v_sliced, size_sliced)

print(res)
# torch.sparse.FloatTensor of size (3,3) with indices:
# tensor([[ 0,  2,  1,  0],
#         [ 0,  1,  0,  0]])
# and values:
# tensor([ 2.,  3.,  4.,  6.])

print(res.to_dense())
# tensor([[ 8.,  0.,  0.],
#         [ 4.,  0.,  0.],
#         [ 0.,  3.,  0.]])

Previous answer for 1-dimentional sparse indices

Here is a (probably sub-optimal and not covering all edge cases) solution, following the intuitions shared in a related open issue (hopefully this feature will be properly covered soon):

# Constructing a sparse tensor a bit more complicated for the sake of demo:
i = torch.LongTensor([[0, 1, 5, 2]])
v = torch.FloatTensor([[1, 3, 0], [5, 7, 0], [9, 9, 9], [1,2,3]])
test1 = torch.sparse.FloatTensor(i, v)

# note: if you directly have sparse `test1`, you can get `i` and `v`:
# i, v = test1._indices(), test1._values()

# Getting the slicing indices:
idx = [1,2]

# Preparing to slice `v` according to `idx`.
# For that, we gather the list of indices `v_idx` such that i[v_idx[k]] == idx[k]:
i_squeeze = i.squeeze()
v_idx = [(i_squeeze == j).nonzero() for j in idx] # <- doesn't seem optimal...
v_idx = torch.cat(v_idx, dim=1)

# Slicing `v` accordingly:
v_sliced = v[v_idx.squeeze()][:,idx]

# Now defining your resulting sparse tensor.
# I'm not sure what kind of indexing you want, so here are 2 possibilities:
# 1) "Dense" indixing:
test1x = torch.sparse.FloatTensor(torch.arange(v_idx.size(1)).long().unsqueeze(0), v_sliced)
print(test1x)
# torch.sparse.FloatTensor of size (3,2) with indices:
#
#  0  1
# [torch.LongTensor of size (1,2)]
# and values:
#
#  7  0
#  2  3
# [torch.FloatTensor of size (2,2)]

# 2) "Sparse" indixing using the original `idx`:
test1x = torch.sparse.FloatTensor(autograd.Variable(torch.LongTensor(idx)).unsqueeze(0), v_sliced)
# note: this indexing would fail if elements of `idx` were not in `i`.
print(test1x)
# torch.sparse.FloatTensor of size (3,2) with indices:
#
#  1  2
# [torch.LongTensor of size (1,2)]
# and values:
#
#  7  0
#  2  3
# [torch.FloatTensor of size (2,2)]


Instruction: Embedding in pytorch

Input: Does Embedding make similar words closer to each other? And do I just need to give to it all the sentences? Or it is just a lookup table and I need to code the model?

Output:
nn.Embedding holds a Tensor of dimension (vocab_size, vector_size), i.e. of the size of the vocabulary x the dimension of each vector embedding, and a method that does the lookup.

When you create an embedding layer, the Tensor is initialised randomly. It is only when you train it when this similarity between similar words should appear. Unless you have overwritten the values of the embedding with a previously trained model, like GloVe or Word2Vec, but that's another story.

So, once you have the embedding layer defined, and the vocabulary defined and encoded (i.e. assign a unique number to each word in the vocabulary) you can use the instance of the nn.Embedding class to get the corresponding embedding.

For example:

import torch
from torch import nn
embedding = nn.Embedding(1000,128)
embedding(torch.LongTensor([3,4]))

will return the embedding vectors corresponding to the word 3 and 4 in your vocabulary. As no model has been trained, they will be random.


Instruction: dimension out of range (expected to be in range of [-2, 1], but got 2)

Input:
Why does the following error popup? What should be in this range and why? What does -2 dimension mean?

RuntimeError: dimension out of range (expected to be in range of [-2, 1], but got 2)

This code will produce the error

import torch

torch.bmm(torch.randn(1000, 784) , torch.randn(784, 10))

Output:
torch.mm:

  Performs a matrix multiplication of the matrices mat1 and mat2.

  If mat1 is a (n×m) tensor, mat2 is a (m×p) tensor, out will be a (n×p)
      tensor.

torch.bmm:

  Performs a batch matrix-matrix product of matrices stored in batch1
  and batch2.

  batch1 and batch2 must be 3-D tensors each containing the same number
  of matrices.

  If batch1 is a (b×n×m) tensor, batch2 is a (b×m×p) tensor, out will be
  a (b×n×p) tensor.

The following code snippet works.

import torch

x = torch.mm(torch.randn(100, 78) , torch.randn(78, 10))
bsize = 16
x = torch.bmm(torch.randn(bsize, 100, 78) , torch.randn(bsize, 78, 10))


Instruction: Dot Product of Tensors with Different Shapes in PyTorch

Input:
p=torch.randn(2,3)
q=torch.randn(3,4,5)

I want to perform dot product, to obtain a result of shape (2,4,5).

How can this be done with PyTorch?

Output:
Two solutions for multi-dimensional matrix multiplications:

Use Tensor.reshape() to get 2-D tensors and use torch.mm();
Use directly torch.einsum().

Demonstration:

import torch

p=torch.randn(2,3)
q=torch.randn(3,4,5)

# Solution 1: Reshaping to use 2-dimensional torch.mm()
res1 = torch.mm(p, q.resize(3, 4 * 5)).resize_(2, 4, 5)
print(res1.shape)
# torch.Size([2, 4, 5])

# Solution 2: Using explicit torch.einsum()
res2 = torch.einsum("ab,bcd->acd", (p, q))
print(res2.shape)
# torch.Size([2, 4, 5])

# Checking if results are equal:
print((res1 == res2).all())
# tensor(1, dtype=torch.uint8)


Instruction: Python loop faster than Pandas

Input:
The below code will show that using python loop is faster than using Pandas. My understanding before I tested it was different. So I'm wondering am I using pandas wrongly for this operation?
The below code shows that Pandas solution is about 7 times slower:

Pandas time  0.0008931159973144531
Loop time    0.0001239776611328125

Code:

import pandas as pd
import numpy as np
import time
import torch

batch_size = 5
classes = 4
raw_target = torch.from_numpy(np.array([1, 0, 3, 2, 0]))
rows = np.array(range(batch_size))

t0 = time.time()
zeros = pd.DataFrame(0, index=range(batch_size), columns=range(classes))
zeros.iloc[[rows, raw_target.numpy()]] = 1
t1 = time.time()

print("Pandas time ", t1-t0)
t0 = time.time()

target = raw_target.numpy()
zeros = np.zeros((batch_size, classes), dtype=np.float64)
for zero, target in zip(zeros, target):
    zero[target] = 1

t1 = time.time()
print("Loop  time  ", t1-t0)

The code uses PyTorch because the actual code where the problem exists uses PyTorch.
What could be better/optimal solution to this example? The resulting matrix is:

[[0. 1. 0. 0.]
 [1. 0. 0. 0.]
 [0. 0. 0. 1.]
 [0. 0. 1. 0.]
 [1. 0. 0. 0.]]

Output:
Depending on your use-case, having everything running through PyTorch could be advantageous (e.g. to keep all computations on the GPU).

The PyTorch-only solution would follow the numpy syntax (i.e. zeros[rows, raw_target] = 1.):

import numpy as np
import torch

batch_size = 5
classes = 4
raw_target = torch.from_numpy(np.array([1, 0, 3, 2, 0]))
rows = torch.range(0, batch_size-1, dtype=torch.int64)

x = torch.zeros((batch_size, classes), dtype=torch.float64)
x[rows, raw_target] = 1.

print(x.detach())
# tensor([[ 0.,  1.,  0.,  0.],
#         [ 1.,  0.,  0.,  0.],
#         [ 0.,  0.,  0.,  1.],
#         [ 0.,  0.,  1.,  0.],
#         [ 1.,  0.,  0.,  0.]], dtype=torch.float64)


Instruction: Minimization and maximization at the same time in PyTorch

Input:
I would like to know how to take gradient steps for the following mathematical operation in PyTorch (A, B and C are PyTorch modules whose parameters do not overlap)

This is somewhat different than the cost function of a Generative Adversarial Network (GAN), so I cannot use examples for GANs off the shelf, and I got stuck while trying to adapt them for the above cost.

One approach I thought of is to construct two optimizers. Optimizer opt1 has the parameters for the modules A and B, and optimizer opt2 has the parameters of module C. One can then:

take a step for minimizing the cost function for C
run the network again with the same input to get the costs (and intermediate outputs) again
take a step with respect to A and B.

I am sure they must be a better way to do this with PyTorch (maybe using some detach operations), possibly without running the network again. Any help is appreciated.

Output:
Yes it is possible without going through the network two times, which is both wasting resources and wrong mathematically, since the weights have changed and so the lost, so you are introducing a delay doing this, which may be interesting but not what you are trying to achieve.
First, create two optimizers just as you said. Compute the loss, and then call backward. At this point, the gradient for the parameters A,B,C have been filled, so now you can just have to call the step method for the optimizer minimizing the loss, but not for the one maximizing it. For the later, you need to reverse the sign of the gradient of the leaf parameter tensor C.
def d(y, x):
    return torch.pow(y.abs(), x + 1)

A = torch.nn.Linear(1,2)
B = torch.nn.Linear(2,3)
C = torch.nn.Linear(2,3)

optimizer1 = torch.optim.Adam((*A.parameters(), *B.parameters()))
optimizer2 = torch.optim.Adam(C.parameters())

x = torch.rand((10, 1))
loss = (d(B(A(x)), x) - d(C(A(x)), x)).sum()

optimizer1.zero_grad()
optimizer2.zero_grad()

loss.backward()
for p in C.parameters():
    if p.grad is not None: # In general, C is a NN, with requires_grad=False for some layers
        p.grad.data.mul_(-1) # Update of grad.data not tracked in computation graph

optimizer1.step()
optimizer2.step()

NB: I have not checked mathematically if the result is correct but I assume it is.


Instruction: How do I add LSTM, GRU or other recurrent layers to a Sequential in PyTorch

Input:
I like using torch.nn.Sequential as in

self.conv_layer = torch.nn.Sequential(
    torch.nn.Conv1d(196, 196, kernel_size=15, stride=4),
    torch.nn.Dropout()
)

But when I want to add a recurrent layer such as torch.nn.GRU it won't work because the output of recurrent layers in PyTorch is a tuple and you need to choose which part of the output you want to further process.

So is there any way to get

self.rec_layer = nn.Sequential(
    torch.nn.GRU(input_size=2, hidden_size=256),
    torch.nn.Linear(in_features=256, out_features=1)
)

to work? For this example, let's say I want to feed torch.nn.GRU(input_size=2, hidden_size=20)(x)[1][-1] (the last hidden state of the last layer) into the following Linear layer.

Output:
I made a module called SelectItem to pick out an element from a tuple or list

class SelectItem(nn.Module):
    def __init__(self, item_index):
        super(SelectItem, self).__init__()
        self._name = 'selectitem'
        self.item_index = item_index

    def forward(self, inputs):
        return inputs[self.item_index]

SelectItem can be used in Sequential to pick out the hidden state:

    net = nn.Sequential(
        nn.GRU(dim_in, dim_out, batch_first=True),
        SelectItem(1)
        )


Instruction: PyTorch: copy_if failed to synchronize: device-side assert triggered

Input:
Getting the following error:

  RuntimeError: copy_if failed to synchronize: device-side assert triggered

when trying to execute:

if torch.nonzero(gt_class_ids > n_classes).size()[0] > 0:

where gt_class_ids is Torch.cuda.LongTensor of size [128] and n_classes = 81.
Running on cuda 9.x

Output:
The following code works for me.

n_classes = 81
gt_class_ids = torch.from_numpy(numpy.random.randint(1, 100, size=128)).long()

if torch.nonzero(gt_class_ids > n_classes).size(0) > 0:
    print('okay')

One suggestion: run the code without using cuda and then you will be able to see the real error message. Sometimes when we run code using cuda, it gives error message having device-side assert triggered which hides the real error message.


Instruction: I want to customise the last layer of VGG 19 architecture for a classification. which will be more useful keras or pytorch?

Input: I want to customise the last layer of VGG 19 architecture for a classification problem. which will be more useful keras or pytorch?

Output:
It heavily depends on what you want to do with it.
While Keras offers different backends, such as TensorFlow or Theano (which in turn can offer you a little more flexibility), and transfers better to production systems,
PyTorch is definitely also easy to implement. Additionally, it offers great scaling on (multi-)GPU systems, since it is trivial to outsource your computations in a PyTorch model. I do not know how easy that is in Keras (never done it, so I genuinely cannot judge).

If you just want to play around with one of the frameworks, it usually boils down to personal preference. I personally prefer PyTorch, due to its more "python-esque" approach to things, but I know many people that prefer Keras because of its clear and simple layout and documentation.

Providing a little more information, or your context, can also potentially increase the quality of the answers you receive.


Instruction: PyTorch 0.4 LSTM: Why does each epoch get slower?

Input:
I have a toy model of a PyTorch 0.4 LSTM on a GPU.  The overall idea of the toy problem is that I define a single 3-vector as an input, and define a rotation matrix R.  The ground truth targets are then a sequence of vectors:  At T0, the input vector; at T1 the input vector rotated by R; at T2 the input rotated by R twice, etc.  (The input is padded the output length with zero-inputs after T1)

The loss is the average L2 difference between ground truth and outputs.  The rotation matrix, construction of the input/output data, and loss functions are probably not of interest, and not shown here.

Never mind that the results are pretty terrible: Why does this become successively slower with each passing epoch?!

I've shown on-GPU information below, but this happens on the CPU as well (only with larger times.)  The time to execute ten epochs of this silly little thing grows rapidly.  It's quite noticeable just watching the numbers scroll by.

epoch:   0,     loss: 0.1753,   time previous: 33:28.616360 time now: 33:28.622033  time delta: 0:00:00.005673
epoch:  10,     loss: 0.2568,   time previous: 33:28.622033 time now: 33:28.830665  time delta: 0:00:00.208632
epoch:  20,     loss: 0.2092,   time previous: 33:28.830665 time now: 33:29.324966  time delta: 0:00:00.494301
epoch:  30,     loss: 0.2663,   time previous: 33:29.324966 time now: 33:30.109241  time delta: 0:00:00.784275
epoch:  40,     loss: 0.1965,   time previous: 33:30.109241 time now: 33:31.184024  time delta: 0:00:01.074783
epoch:  50,     loss: 0.2232,   time previous: 33:31.184024 time now: 33:32.556106  time delta: 0:00:01.372082
epoch:  60,     loss: 0.1258,   time previous: 33:32.556106 time now: 33:34.215477  time delta: 0:00:01.659371
epoch:  70,     loss: 0.2237,   time previous: 33:34.215477 time now: 33:36.173928  time delta: 0:00:01.958451
epoch:  80,     loss: 0.1076,   time previous: 33:36.173928 time now: 33:38.436041  time delta: 0:00:02.262113
epoch:  90,     loss: 0.1194,   time previous: 33:38.436041 time now: 33:40.978748  time delta: 0:00:02.542707
epoch: 100,     loss: 0.2099,   time previous: 33:40.978748 time now: 33:43.844310  time delta: 0:00:02.865562

The model:

class Sequence(torch.nn.Module):
def __init__ (self):
    super(Sequence, self).__init__()

    self.lstm1 = nn.LSTM(3,30)
    self.lstm2 = nn.LSTM(30,300)
    self.lstm3 = nn.LSTM(300,30)
    self.lstm4 = nn.LSTM(30,3)

    self.hidden1 = self.init_hidden(dim=30)
    self.hidden2 = self.init_hidden(dim=300)
    self.hidden3 = self.init_hidden(dim=30)
    self.hidden4 = self.init_hidden(dim=3)

    self.dense   = torch.nn.Linear(30, 3)
    self.relu    = nn.LeakyReLU()

def init_hidden(self, dim):
    return (torch.zeros(1, 1, dim).to(device)  ,torch.zeros(1, 1, dim).to(device)  )

def forward(self, inputs):
    out1, self.hidden1 = self.lstm1(inputs, self.hidden1)
    out2, self.hidden2 = self.lstm2(out1,   self.hidden2)
    out3, self.hidden3 = self.lstm3(out2,   self.hidden3)
    #out4, self.hidden4 = self.lstm4(out3,   self.hidden4)

    # This is intended to act as a dense layer on the output of the LSTM
    out4               = self.relu(self.dense(out3))

    return out4

The training loop:

sequence = Sequence().to(device)

criterion = L2_Loss()
optimizer = torch.optim.Adam(sequence.parameters())
_, _, _, R = getRotation(np.pi/27, np.pi/26, np.pi/25)

losses = []
date1 = datetime.datetime.now()
for epoch in range(1001):
    # Define input as a Variable-- each row of 3 is a vector, a distinct input
    # Define target directly from input by applicatin of rotation vector
    # Define predictions by running input through model

    inputs       = getInput(25)
    targets      = getOutput(inputs, R)

    inputs       = torch.cat(inputs).view(len(inputs), 1, -1).to(device)
    targets      = torch.cat(targets).view(len(targets), 1, -1).to(device)

    target_preds = sequence(inputs)
    target_preds = target_preds.view(len(target_preds), 1, -1)
    loss = criterion(targets, target_preds).to(device)

    losses.append(loss.data[0])
    if (epoch % 10 == 0):
        date2 = datetime.datetime.now()
        print("epoch: %3d, \tloss: %6.4f, \ttime previous: %s\ttime now: %s\ttime delta: %s" % (epoch, loss.data[0], date1.strftime("%M:%S.%f"), date2.strftime("%M:%S.%f"), date2 - date1))
        date1 = date2
    # Zero out the grads, run the loss backward, and optimize on the grads
    optimizer.zero_grad()
    loss.backward(retain_graph=True)
    optimizer.step()

Output:
Short answer:  Because we did not detach the hidden layers, and therefore the system kept backpropagating farther and father back through time, taking up more memory and requiring more time.

Long answer:  This answer is meant to work without teacher forcing.  "Teacher forcing" is when all inputs at all time-steps are "ground truth" input values.  In contrast, without teacher forcing, the input of each time step is the output from the previous time step, no matter how early in the training regime (and therefore, how wildly erratic) that data might be.

This is a bit of a manual operation in PyTorch, requiring us to keep track of not only the output, but the hidden state of the network at each step, so we can provide it to the next.  Detachment has to happen, not at every time step, but at the beginning of each sequence.  A method that seems to work is to define a "detach" method as part of the Sequence model (which will manually detach all the hidden layers), and call it explicitly
after the optimizer.step().

This prevents the gradual accumulation of the hidden states, prevents the gradual slowdown, and still seems to train the network.

I cannot truly vouch for it because I have only employed it on a toy model, not a real problem.

Note 1: There are probably better ways to factor the initialization of the network and use that instead of a manual detach, as well.

Note2:  The loss.backward(retain_graph=True) statement retains the graph because error messages suggested it.  Once the detach is enacted, that warning disappears.

I leave this answer un-accepted in the hopes that someone knowledgeable will add their expertise.


Instruction: System hangs after first epoch training in pytorch

Input:
So, I was trying to train on ResNet model in PyTorch using the ImageNet example in the GitHub repository.

Here's what my train method looks like (it is almost similar to that in example)

def train(train_loader, model, criterion, optimizer, epoch):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    args = get_args()

    # switch to train mode
    model.train()

    end = time.time()

    for i, (input, target) in enumerate(train_loader):
        print(i)
        # data loading time
        data_time.update(time.time() - end)

        if cuda:
            target = target.cuda(async = True)
            input_var = torch.autograd.Variable(input).cuda()
        else:
            input_var = torch.autograd.Variable(input)

        target_var = torch.autograd.Variable(target)

        # compute output
        output = model(input_var)
        loss = criterion(output, target_var)

        # measure accuracy and record loss
        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))
        losses.update(loss.item(), input.size(0))
        top1.update(prec1.item(), input.size(0))
        # top5.update(prec5.item(), input.size(0))

        # compute gradient and do optimizer step
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        #measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        # print to console and write logs to tensorboard
        if i % args.print_freq == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, top1=top1, top5=top5))
            niter = epoch * len(train_loader) + i
            # writer.add_scalar('Train/Loss', losses.val, niter)
            # writer.add_scalar('Train/Prec@1', top1.val, niter)
            # writer.add_scalar('Train/Prec@5', top5.val, niter)

System Information:
GPU: Nvidia Titan XP
Memory: 32 Gb

PyTorch: 0.4.0

When I run this code, training starts with epoch 0

Epoch: [0][0/108]   Time 5.644 (5.644)  Data 1.929 (1.929)  Loss 6.9052 (6.9052)    Prec@1 0.000 (0.000)

And then the remote server automatically disconnects. It happened five times.

And this is the data loader:

#Load the Data --> TRAIN
    traindir = 'train'
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])
    train_dataset = datasets.ImageFolder(traindir, transforms.Compose([
            transforms.RandomResizedCrop(224),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))
    train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
        pin_memory=cuda
    )

    # Load the data --> Validation
    valdir = 'valid'
    valid_loader = torch.utils.data.DataLoader(
        datasets.ImageFolder(valdir, transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            normalize,
        ])),
        batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers,
        pin_memory=cuda
    )

    if args.evaluate:
        validate(valid_loader, model, criterion, epoch=0)
        return

    # Start
    for epoch in range(args.start_epoch, args.epochs):
        adjust_learning_rate(optimizer, epoch)

        # train for epoch
        train(train_loader, model, criterion, optimizer, epoch)

        # evaluate on valid
        prec1 = validate(valid_loader, model, criterion, epoch)

        # remember best prec1 and save checkpoint
        is_best = prec1 > best_prec1
        best_prec1 = max(prec1, best_prec1)
        save_checkpoint({
            'epoch': epoch + 1,
            'arch': args.arch,
            'state_dict': model.state_dict(),
            'best_prec1': best_prec1,
            'optimizer': optimizer.state_dict()
        }, is_best)

With this params for the loader:

args.num_workers = 4
args.batch_size = 32
pin_memory = torch.cuda.is_available()

Is there something wrong in my approach?

Output:
seems a bug in pytorch's dataloader.
try args.num_workers = 0