Instruction: Spyder just "kernel died,restarting" when I run net.forward in pytorch

Input:
I am using PYTHON IDE spyder3.2.1 in anaconda2, with python2.7, ubuntu14.04

code is just simple as follows:

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

"""
input:[batch_size,in_channel,height,width]
kernel:[out_channel,in_channel,kh,kw]
"""
class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # 1 input image channel, 6 output channels, 5x5 square convolution
        # kernel
        self.conv1 = nn.Conv2d(1, 6, 5)
        #(28-5+1)/2=12
        self.conv2 = nn.Conv2d(6, 16, 5)
        #(12-5+1)/2=4
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        # If the size is a square you can only specify a single number
        print "after conv1 size is {}".format(x.size())
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        print "after conv2 size is {}".format(x.size())
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

net = Net()
print(net)

print "hello wrold"
input = Variable(torch.Tensor(np.random.randint(1,10,size=(1,1,32,32))))
print net.forward(input)

when I use jupyter notebook or console, it runs normally, it means the code has no error.
But when I use it from Spyder, it just hangs likes this:

As I am not allowed to embed the error image: it says "Kernel died, restarting"

I just try to shrink my code to find where the problem is.

The problem is absolutely around the very start F.max_pool2d(), in the forward method function.

But after I searched a lot, I still have no idea how to fix it.

conda list:

alabaster                 0.7.10                   py27_0
anaconda                  4.4.0               np112py27_0
anaconda-client           1.6.3                    py27_0
anaconda-navigator        1.6.2                    py27_0
anaconda-project          0.6.0                    py27_0
asn1crypto                0.22.0                   py27_0
astroid                   1.4.9                    py27_0
astroid                   1.5.3                     <pip>
astropy                   1.3.2               np112py27_0
Babel                     2.5.0                     <pip>
babel                     2.4.0                    py27_0
backports                 1.0                      py27_0
backports.functools-lru-cache 1.4                       <pip>
backports.weakref         1.0rc1                    <pip>
backports_abc             0.5                      py27_0
beautifulsoup4            4.6.0                    py27_0
bitarray                  0.8.1                    py27_0
blaze                     0.10.1                   py27_0
bleach                    1.5.0                    py27_0
bleach                    2.0.0                     <pip>
bokeh                     0.12.5                   py27_1
boto                      2.46.1                   py27_0
bottleneck                1.2.1               np112py27_0
cairo                     1.14.8                        0
cdecimal                  2.3                      py27_2
certifi                   2017.7.27.1               <pip>
cffi                      1.10.0                   py27_0
chardet                   3.0.3                    py27_0
chardet                   3.0.4                     <pip>
click                     6.7                      py27_0
cloudpickle               0.2.2                    py27_0
clyent                    1.2.2                    py27_0
colorama                  0.3.9                    py27_0
conda                     4.3.21                   py27_0
conda-env                 2.6.0                         0
configparser              3.5.0                    py27_0
contextlib2               0.5.5                    py27_0
cryptography              1.8.1                    py27_0
curl                      7.52.1                        0
cycler                    0.10.0                   py27_0
cython                    0.25.2                   py27_0
cytoolz                   0.8.2                    py27_0
dask                      0.14.3                   py27_1
datashape                 0.5.4                    py27_0
dbus                      1.10.10                       0
decorator                 4.0.11                   py27_0
distributed               1.16.3                   py27_0
docutils                  0.14                      <pip>
docutils                  0.13.1                   py27_0
entrypoints               0.2.2                    py27_1
entrypoints               0.2.3                     <pip>
enum34                    1.1.6                    py27_0
et_xmlfile                1.0.1                    py27_0
expat                     2.1.0                         0
fastcache                 1.0.2                    py27_1
flask                     0.12.2                   py27_0
flask-cors                3.0.2                    py27_0
fontconfig                2.12.1                        3
freetype                  2.5.5                         2
funcsigs                  1.0.2                    py27_0
functools32               3.2.3.2                  py27_0
futures                   3.1.1                    py27_0
get_terminal_size         1.0.0                    py27_0
gevent                    1.2.1                    py27_0
glib                      2.50.2                        1
greenlet                  0.4.12                   py27_0
grin                      1.2.1                    py27_3
gst-plugins-base          1.8.0                         0
gstreamer                 1.8.0                         0
h5py                      2.7.0               np112py27_0
harfbuzz                  0.9.39                        2
hdf5                      1.8.17                        1
heapdict                  1.0.0                    py27_1
html5lib                  0.999999999               <pip>
html5lib                  0.999                    py27_0
icu                       54.1                          0
idna                      2.5                      py27_0
idna                      2.6                       <pip>
imagesize                 0.7.1                    py27_0
ipaddress                 1.0.18                   py27_0
ipykernel                 4.6.1                    py27_0
ipython                   5.3.0                    py27_0
ipython_genutils          0.2.0                    py27_0
ipywidgets                6.0.0                    py27_0
isort                     4.2.5                    py27_0
isort                     4.2.15                    <pip>
itsdangerous              0.24                     py27_0
jbig                      2.1                           0
jdcal                     1.3                      py27_0
jedi                      0.10.2                   py27_2
jinja2                    2.9.6                    py27_0
jpeg                      9b                            0
jsonschema                2.6.0                    py27_0
jupyter                   1.0.0                    py27_3
jupyter-client            5.1.0                     <pip>
jupyter_client            5.0.1                    py27_0
jupyter_console           5.1.0                    py27_0
jupyter_core              4.3.0                    py27_0
lazy-object-proxy         1.2.2                    py27_0
lazy-object-proxy         1.3.1                     <pip>
libffi                    3.2.1                         1
libgcc                    4.8.5                         2
libgfortran               3.0.0                         1
libiconv                  1.14                          0
libpng                    1.6.27                        0
libsodium                 1.0.10                        0
libtiff                   4.0.6                         3
libtool                   2.4.2                         0
libxcb                    1.12                          1
libxml2                   2.9.4                         0
libxslt                   1.1.29                        0
llvmlite                  0.18.0                   py27_0
locket                    0.2.0                    py27_1
lxml                      3.7.3                    py27_0
Markdown                  2.6.8                     <pip>
markupsafe                0.23                     py27_2
MarkupSafe                1.0                       <pip>
matplotlib                2.0.2               np112py27_0
mccabe                    0.6.1                     <pip>
mistune                   0.7.4                    py27_0
mkl                       2017.0.1                      0
mkl-service               1.1.2                    py27_3
mock                      2.0.0                     <pip>
mpmath                    0.19                     py27_1
msgpack-python            0.4.8                    py27_0
multipledispatch          0.4.9                    py27_0
navigator-updater         0.1.0                    py27_0
nbconvert                 5.2.1                     <pip>
nbconvert                 5.1.1                    py27_0
nbformat                  4.4.0                     <pip>
nbformat                  4.3.0                    py27_0
networkx                  1.11                     py27_0
nltk                      3.2.3                    py27_0
nose                      1.3.7                    py27_1
notebook                  5.0.0                    py27_0
numba                     0.33.0              np112py27_0
numexpr                   2.6.2               np112py27_0
numpy                     1.12.1                   py27_0
numpy                     1.13.1                    <pip>
numpydoc                  0.6.0                    py27_0
numpydoc                  0.7.0                     <pip>
odo                       0.5.0                    py27_1
olefile                   0.44                     py27_0
opencv                    1.0.1                     <pip>
opencv-python             3.3.0.9                   <pip>
openpyxl                  2.4.7                    py27_0
openssl                   1.0.2l                        0
packaging                 16.8                     py27_0
pandas                    0.20.1              np112py27_0
pandocfilters             1.4.1                    py27_0
pandocfilters             1.4.2                     <pip>
pango                     1.40.3                        1
partd                     0.3.8                    py27_0
path.py                   10.3.1                   py27_0
pathlib2                  2.3.0                     <pip>
pathlib2                  2.2.1                    py27_0
patsy                     0.4.1                    py27_0
pbr                       3.1.1                     <pip>
pcre                      8.39                          1
pep8                      1.7.0                    py27_0
pexpect                   4.2.1                    py27_0
pickleshare               0.7.4                    py27_0
pillow                    4.1.1                    py27_0
pip                       9.0.1                    py27_1
pixman                    0.34.0                        0
ply                       3.10                     py27_0
prompt_toolkit            1.0.14                   py27_0
protobuf                  3.3.0                     <pip>
psutil                    5.2.2                    py27_0
ptyprocess                0.5.1                    py27_0
py                        1.4.33                   py27_0
pycairo                   1.10.0                   py27_0
pycodestyle               2.3.1                     <pip>
pycosat                   0.6.2                    py27_0
pycparser                 2.17                     py27_0
pycrypto                  2.6.1                    py27_6
pycurl                    7.43.0                   py27_2
pyflakes                  1.5.0                    py27_0
pyflakes                  1.6.0                     <pip>
pygments                  2.2.0                    py27_0
pylint                    1.6.4                    py27_1
pylint                    1.7.2                     <pip>
pylzma                    0.4.9.post0               <pip>
pyodbc                    4.0.16                   py27_0
PyOpenGL                  3.1.0                     <pip>
pyopenssl                 17.0.0                   py27_0
pyparsing                 2.1.4                    py27_0
pyqt                      5.6.0                    py27_2
pytables                  3.3.0               np112py27_0
pytest                    3.0.7                    py27_0
python                    2.7.13                        0
python-dateutil           2.6.1                     <pip>
python-dateutil           2.6.0                    py27_0
pytz                      2017.2                   py27_0
pywavelets                0.5.2               np112py27_0
pyyaml                    3.12                     py27_0
pyzmq                     16.0.2                   py27_0
qt                        5.6.2                         4
qtawesome                 0.4.4                    py27_0
qtconsole                 4.3.0                    py27_0
qtconsole                 4.3.1                     <pip>
qtpy                      1.2.1                    py27_0
QtPy                      1.3.1                     <pip>
readline                  6.2                           2
requests                  2.18.4                    <pip>
requests                  2.14.2                   py27_0
rope                      0.10.7                    <pip>
rope                      0.9.4                    py27_1
ruamel_yaml               0.11.14                  py27_1
scandir                   1.5                      py27_0
scikit-image              0.13.0              np112py27_0
scikit-learn              0.18.1              np112py27_1
scipy                     0.19.0              np112py27_0
seaborn                   0.7.1                    py27_0
setuptools                27.2.0                   py27_0
setuptools                36.2.7                    <pip>
simplegeneric             0.8.1                    py27_1
singledispatch            3.4.0.3                  py27_0
sip                       4.18                     py27_0
six                       1.10.0                   py27_0
snowballstemmer           1.2.1                    py27_0
sortedcollections         0.5.3                    py27_0
sortedcontainers          1.5.7                    py27_0
Sphinx                    1.6.3                     <pip>
sphinx                    1.5.6                    py27_0
sphinxcontrib-websupport  1.0.1                     <pip>
spyder                    3.2.1                     <pip>
spyder                    3.1.4                    py27_0
sqlalchemy                1.1.9                    py27_0
sqlite                    3.13.0                        0
ssl_match_hostname        3.4.0.2                  py27_1
statsmodels               0.8.0               np112py27_0
subprocess32              3.2.7                    py27_0
sympy                     1.0                      py27_0
tblib                     1.3.2                    py27_0
tensorflow                1.2.1                     <pip>
tensorlayer               1.6.0                     <pip>
terminado                 0.6                      py27_0
testpath                  0.3.1                     <pip>
testpath                  0.3                      py27_0
tk                        8.5.18                        0
toolz                     0.8.2                    py27_0
torch-0.2.0               1                         <pip>
torchvision               0.1.9                     <pip>
tornado                   4.5.1                    py27_0
tqdm                      4.15.0                    <pip>
traitlets                 4.3.2                    py27_0
typing                    3.6.2                     <pip>
unicodecsv                0.14.1                   py27_0
unixodbc                  2.3.4                         0
urllib3                   1.22                      <pip>
wcwidth                   0.1.7                    py27_0
webencodings              0.5.1                     <pip>
werkzeug                  0.12.2                   py27_0
wheel                     0.29.0                   py27_0
widgetsnbextension        2.0.0                    py27_0
wrapt                     1.10.11                   <pip>
wrapt                     1.10.10                  py27_0
xlrd                      1.0.0                    py27_0
xlsxwriter                0.9.6                    py27_0
xlwt                      1.2.0                    py27_0
xz                        5.2.2                         1
yaml                      0.1.6                         0
zeromq                    4.1.5                         0
zict                      0.1.2                    py27_0
zlib                      1.2.8                         3

Output:
(Spyder developer here) I don't know why this is happening, but I opened an issue in our issue tracker so we don't forget to take a look at it in the future.

Update: This problem was solved by the OP by updating PyTorch.


Instruction: Pytorch install with anaconda error

Input:
I get this error:

C:\Users>conda install pytorch torchvision -c soumith
Fetching package metadata .............

PackageNotFoundError: Package missing in current win-64 channels:
  - pytorch

I got conda install pytorch torchvision -c soumith from Pytorch official website and I have OSX/conda/3.6/none for settings on Pytorch site(should be correct). I am new to conda, any tips how to solve this?

Output:
Use the following commands to install pytorch on windows

for Windows 10 and Windows Server 2016, CUDA 8

conda install -c peterjc123 pytorch cuda80

for Windows 10 and Windows Server 2016, CUDA 9

conda install -c peterjc123 pytorch cuda90

for Windows 7/8/8.1 and Windows Server 2008/2012, CUDA 8

conda install -c peterjc123 pytorch_legacy cuda80


Instruction: How should I save the model of PyTorch if I want it loadable by OpenCV dnn module

Input:
I train a simple classification model by PyTorch and load it by opencv3.3, but it throw exception and say

  OpenCV Error: The function/feature is not implemented (Unsupported Lua type) in readObject, file
  /home/ramsus/Qt/3rdLibs/opencv/modules/dnn/src/torch/torch_importer.cpp,
  line 797
  /home/ramsus/Qt/3rdLibs/opencv/modules/dnn/src/torch/torch_importer.cpp:797:
  error: (-213) Unsupported Lua type in function readObject

Model definition

class conv_block(nn.Module):
    def __init__(self, in_filter, out_filter, kernel):
        super(conv_block, self).__init__()

        self.conv1 = nn.Conv2d(in_filter, out_filter, kernel, 1, (kernel - 1)//2)
        self.batchnorm = nn.BatchNorm2d(out_filter)
        self.maxpool = nn.MaxPool2d(2, 2)

    def forward(self, x):
        x = self.conv1(x)
        x = self.batchnorm(x)
        x = F.relu(x)
        x = self.maxpool(x)

        return x

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.conv1 = conv_block(3, 6, 3)
        self.conv2 = conv_block(6, 16, 3)
        self.fc1 = nn.Linear(16 * 8 * 8, 120)
        self.bn1 = nn.BatchNorm1d(120)
        self.fc2 = nn.Linear(120, 84)
        self.bn2 = nn.BatchNorm1d(84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size()[0], -1)
        x = F.relu(self.bn1(self.fc1(x)))
        x = F.relu(self.bn2(self.fc2(x)))
        x = self.fc3(x)
        return x

This model use Conv2d, ReLU, BatchNorm2d, MaxPool2d and Linear layer only, every layers are supported by opencv3.3

I save it by state_dict

torch.save(net.state_dict(), 'cifar10_model')

Load it by c++ as

std::string const model_file("/home/some_folder/cifar10_model");

std::cout<<"read net from torch"<<std::endl;
dnn::Net net = dnn::readNetFromTorch(model_file);

I guess I save the model with the wrong way, what is the proper way to save the model of PyTorch in order to load using OpenCV? Thanks

Edit :

I use another way to save the model, but it cannot be loaded either

torch.save(net, 'cifar10_model.net')

Is this a bug?Or I am doing something wrong?

Output: I found the answer, opencv3.3 do not support PyTorch (https://github.com/pytorch/pytorch) but pytorch (https://github.com/hughperkins/pytorch), it is a big surprise, I never know there are another version of pytorch exist(looks like a dead project, long time haven't updated), I hope they could mention which pytorch they support on wiki.


Instruction: Why does autograd not produce gradient for intermediate variables?

Input:
trying to wrap my head around how gradients are represented and how autograd works:

import torch
from torch.autograd import Variable

x = Variable(torch.Tensor([2]), requires_grad=True)
y = x * x
z = y * y

z.backward()

print(x.grad)
#Variable containing:
#32
#[torch.FloatTensor of size 1]

print(y.grad)
#None

Why does it not produce a gradient for y? If y.grad = dz/dy, then shouldn't it at least produce a variable like y.grad = 2*y?

Output:
  By default, gradients are only retained for leaf variables. non-leaf variables' gradients are not retained to be inspected later. This was
  done by design, to save memory.

-soumith chintala

See: https://discuss.pytorch.org/t/why-cant-i-see-grad-of-an-intermediate-variable/94

Option 1:

Call y.retain_grad()

x = Variable(torch.Tensor([2]), requires_grad=True)
y = x * x
z = y * y

y.retain_grad()

z.backward()

print(y.grad)
#Variable containing:
# 8
#[torch.FloatTensor of size 1]

Source: https://discuss.pytorch.org/t/why-cant-i-see-grad-of-an-intermediate-variable/94/16

Option 2:

Register a hook, which is basically a function called when that gradient is calculated. Then you can save it, assign it, print it, whatever...

from __future__ import print_function
import torch
from torch.autograd import Variable

x = Variable(torch.Tensor([2]), requires_grad=True)
y = x * x
z = y * y

y.register_hook(print) ## this can be anything you need it to be

z.backward()

output:

Variable containing:  8 [torch.FloatTensor of size 1

Source: https://discuss.pytorch.org/t/why-cant-i-see-grad-of-an-intermediate-variable/94/2

Also see: https://discuss.pytorch.org/t/why-cant-i-see-grad-of-an-intermediate-variable/94/7


Instruction: pytorch inception model outputs the wrong label for every input image

Input:
For the pytorch models I found this tutorial explaining how to classify an image. I tried to apply the same prcedure for an inception model. However the model fails for every image I load in

Code:

# some people need these three lines to make it work
#from torchvision.models.inception import model_urls
#name = 'inception_v3_google'
#model_urls[name] = model_urls[name].replace('https://', 'http://')

from torch.autograd import Variable
import torchvision
import requests
from torchvision import models, transforms
from PIL import Image
import io
from PIL import Image

LABELS_URL = 'https://s3.amazonaws.com/outcome-blog/imagenet/labels.json'

# cat
IMG_URL1 = 'http://farm2.static.flickr.com/1029/762542019_4f197a0de5.jpg'

# dog
IMG_URL2 = 'http://farm3.static.flickr.com/2314/2518519714_98b01968ee.jpg'

# lion
IMG_URL3 = 'http://farm1.static.flickr.com/62/218998565_62930f10fc.jpg'

labels = {int(key):value for (key, value)
          in requests.get(LABELS_URL).json().items()}

model = torchvision.models.inception_v3(pretrained=True)
model.training = False
model.transform_input = False

def predict_url_img(url):
    response = requests.get(url)
    img_pil = Image.open(io.BytesIO(response.content))

    normalize = transforms.Normalize(
       mean=[0.485, 0.456, 0.406],
       std=[0.229, 0.224, 0.225]
    )
    preprocess = transforms.Compose([
       transforms.Scale(256),
       transforms.CenterCrop(299),
       transforms.ToTensor(),
       normalize
    ])

    img_tensor = preprocess(img_pil)
    img_tensor.unsqueeze_(0)
    img_variable = Variable(img_tensor)
    fc_out = model(img_variable)
    print("prediction:", labels[fc_out.data.numpy().argmax()])

predict_url_img(IMG_URL1)
predict_url_img(IMG_URL2)
predict_url_img(IMG_URL3)

As output I get this:

  ('prediction:', u"plunger, plumber's helper")

  ('prediction:', u'plastic bag')

  ('prediction:', u"plunger, plumber's helper")

Output: I found out that one needs to call model.eval() before applying the model. Because of the batch normalisations and also dropout layers, the model bahaves differently for training and testing.


Instruction: How to do scatter and gather operations in numpy?

Input:
I want to implement scatter and gather operations of Tensorflow or PyTorch in Numpy. I have been scratching my head for a while. Any pointers are greatly appreciated!

PyTorch Scatter
PyTorch Gather

Output:
The scatter method turned out to be way more work than I anticipated. I did not find any ready made function in NumPy for it. I'm sharing it here in the interest of anyone who may need to implement it with NumPy.
(p.s. self is the destination or output of the method.)

def scatter_numpy(self, dim, index, src):
    """
    Writes all values from the Tensor src into self at the indices specified in the index Tensor.

    :param dim: The axis along which to index
    :param index: The indices of elements to scatter
    :param src: The source element(s) to scatter
    :return: self
    """
    if index.dtype != np.dtype('int_'):
        raise TypeError("The values of index must be integers")
    if self.ndim != index.ndim:
        raise ValueError("Index should have the same number of dimensions as output")
    if dim >= self.ndim or dim < -self.ndim:
        raise IndexError("dim is out of range")
    if dim < 0:
        # Not sure why scatter should accept dim < 0, but that is the behavior in PyTorch's scatter
        dim = self.ndim + dim
    idx_xsection_shape = index.shape[:dim] + index.shape[dim + 1:]
    self_xsection_shape = self.shape[:dim] + self.shape[dim + 1:]
    if idx_xsection_shape != self_xsection_shape:
        raise ValueError("Except for dimension " + str(dim) +
                         ", all dimensions of index and output should be the same size")
    if (index >= self.shape[dim]).any() or (index < 0).any():
        raise IndexError("The values of index must be between 0 and (self.shape[dim] -1)")

    def make_slice(arr, dim, i):
        slc = [slice(None)] * arr.ndim
        slc[dim] = i
        return slc

    # We use index and dim parameters to create idx
    # idx is in a form that can be used as a NumPy advanced index for scattering of src param. in self
    idx = [[*np.indices(idx_xsection_shape).reshape(index.ndim - 1, -1),
            index[make_slice(index, dim, i)].reshape(1, -1)[0]] for i in range(index.shape[dim])]
    idx = list(np.concatenate(idx, axis=1))
    idx.insert(dim, idx.pop())

    if not np.isscalar(src):
        if index.shape[dim] > src.shape[dim]:
            raise IndexError("Dimension " + str(dim) + "of index can not be bigger than that of src ")
        src_xsection_shape = src.shape[:dim] + src.shape[dim + 1:]
        if idx_xsection_shape != src_xsection_shape:
            raise ValueError("Except for dimension " +
                             str(dim) + ", all dimensions of index and src should be the same size")
        # src_idx is a NumPy advanced index for indexing of elements in the src
        src_idx = list(idx)
        src_idx.pop(dim)
        src_idx.insert(dim, np.repeat(np.arange(index.shape[dim]), np.prod(idx_xsection_shape)))
        self[idx] = src[src_idx]

    else:
        self[idx] = src

    return self

There could be a simpler solution for gather, but this is what I settled on:  (here self is the ndarray that the values are gathered from.)

def gather_numpy(self, dim, index):
    """
    Gathers values along an axis specified by dim.
    For a 3-D tensor the output is specified by:
        out[i][j][k] = input[index[i][j][k]][j][k]  # if dim == 0
        out[i][j][k] = input[i][index[i][j][k]][k]  # if dim == 1
        out[i][j][k] = input[i][j][index[i][j][k]]  # if dim == 2

    :param dim: The axis along which to index
    :param index: A tensor of indices of elements to gather
    :return: tensor of gathered values
    """
    idx_xsection_shape = index.shape[:dim] + index.shape[dim + 1:]
    self_xsection_shape = self.shape[:dim] + self.shape[dim + 1:]
    if idx_xsection_shape != self_xsection_shape:
        raise ValueError("Except for dimension " + str(dim) +
                         ", all dimensions of index and self should be the same size")
    if index.dtype != np.dtype('int_'):
        raise TypeError("The values of index must be integers")
    data_swaped = np.swapaxes(self, 0, dim)
    index_swaped = np.swapaxes(index, 0, dim)
    gathered = np.choose(index_swaped, data_swaped)
    return np.swapaxes(gathered, 0, dim)


Instruction: what is the equivalent of theano.tensor.clip in pytorch?

Input: I want to clip my tensor (not gradient) values to some range. Is there any function in pytorch like there is a function theano.tensor.clip() in theano?

Output: The function you are searching for is called torch.clamp. You can find the documentation here


Instruction: How do I write a PyTorch sequential model?

Input:
How do I write a sequential model in PyTorch, just like what we can do with Keras? I tried:
import torch
import torch.nn as nn

net = nn.Sequential()
net.add(nn.Linear(3, 4))
net.add(nn.Sigmoid())
net.add(nn.Linear(4, 1))
net.add(nn.Sigmoid())
net.float()

But I get the error:

AttributeError: 'Sequential' object has no attribute 'add'

Output:
As described by the correct answer, this is what it would look as a sequence of arguments:
device = torch.device('cpu')
if torch.cuda.is_available():
    device = torch.device('cuda')

net = nn.Sequential(
      nn.Linear(3, 4),
      nn.Sigmoid(),
      nn.Linear(4, 1),
      nn.Sigmoid()
      ).to(device)

print(net)

Sequential(
  (0): Linear(in_features=3, out_features=4, bias=True)
  (1): Sigmoid()
  (2): Linear(in_features=4, out_features=1, bias=True)
  (3): Sigmoid()
  )


Instruction: Pytorch Pre-trained RESNET18 Model

Input: I have trained a pre-trained RESNET18 model in pytorch and saved it. While testing the model is giving different accuracy for different mini-batch size. Does anyone know why?

Output:
Yes, I think so.
RESNET contains batch normalisation layers. At evaluation time you need to fix these; otherwise the running means are continuously being adjusted after processing each batch hence giving you different accuracy .

Try setting:

model.eval()

before evaluation. Note before getting back into training, call model.train().


Instruction: How to train pytorch model with numpy data and batch size?

Input:
I am learning the basics of pytorch and thought to create a simple 4 layer nerual network with dropout to train IRIS dataset for classification. After refering to many tutorials I wrote this code.

import pandas as pd
from sklearn.datasets import load_iris
import torch
from torch.autograd import Variable

epochs=300
batch_size=20
lr=0.01

#loading data as numpy array
data = load_iris()
X=data.data
y=pd.get_dummies(data.target).values

#convert to tensor
X= Variable(torch.from_numpy(X), requires_grad=False)
y=Variable(torch.from_numpy(y), requires_grad=False)
print(X.size(),y.size())

#neural net model
model = torch.nn.Sequential(
    torch.nn.Linear(4, 10),
    torch.nn.ReLU(),
    torch.nn.Dropout(),
    torch.nn.Linear(10, 5),
    torch.nn.ReLU(),
    torch.nn.Dropout(),
    torch.nn.Linear(5, 3),
    torch.nn.Softmax()
)

print(model)

# Loss and Optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
loss_func = torch.nn.CrossEntropyLoss()

for i in range(epochs):
    # Forward pass
    y_pred = model(X)

    # Compute and print loss.
    loss = loss_func(y_pred, y)
    print(i, loss.data[0])

    # Before the backward pass, use the optimizer object to zero all of the
    # gradients for the variables it will update (which are the learnable weights
    # of the model)
    optimizer.zero_grad()

    # Backward pass
    loss.backward()

    # Calling the step function on an Optimizer makes an update to its parameters
    optimizer.step()

There are currently two problems I am facing.

I want to set a batch size of 20. How should I do this?
At this step y_pred = model(X) its showing this error

Error

 TypeError: addmm_ received an invalid combination of arguments - got (int, int, torch.DoubleTensor, torch.FloatTensor), but expected one of:
 * (torch.DoubleTensor mat1, torch.DoubleTensor mat2)
 * (torch.SparseDoubleTensor mat1, torch.DoubleTensor mat2)
 * (float beta, torch.DoubleTensor mat1, torch.DoubleTensor mat2)
 * (float alpha, torch.DoubleTensor mat1, torch.DoubleTensor mat2)
 * (float beta, torch.SparseDoubleTensor mat1, torch.DoubleTensor mat2)
 * (float alpha, torch.SparseDoubleTensor mat1, torch.DoubleTensor mat2)
 * (float beta, float alpha, torch.DoubleTensor mat1, torch.DoubleTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, torch.DoubleTensor, !torch.FloatTensor!)
 * (float beta, float alpha, torch.SparseDoubleTensor mat1, torch.DoubleTensor mat2)
      didn't match because some of the arguments have invalid types: (int, int, !torch.DoubleTensor!, !torch.FloatTensor!)

Output:
  I want to set a batch size of 20. How should I do this?

For data processing and loading, PyTorch provide two classes, one is Dataset, which is used to represent your dataset. Specifically, Dataset provide the interface to get one sample from the whole dataset using the sample index.

But Dataset is not enough, for large dataset, we need to do batch processing. So PyTorch provide a second class Dataloader, which is used to generate batches from the Dataset given the batch size and other parameters.

For your specific case, I think you should try TensorDataset. Then use a Dataloader to set batch size to 20. Just look through the PyTorch official examples to get a sense how to do it.

  At this step y_pred = model(X) its showing this error

The error message is pretty informative. Your input X to the model is type DoubleTensor. But your model parameters have type FloatTensor. In PyTorch, you can not do operation between Tensors of different types. What you should do is replace the line

X= Variable(torch.from_numpy(X), requires_grad=False)

with

X= Variable(torch.from_numpy(X).float(), requires_grad=False)

Now, X has type FloatTensor, the error message should disappear.

Also, as a gentle reminder, there are pretty much materials on the Internet about your question which can sufficiently solve your problem. You should try hard to solve it by yourself.


Instruction: How does pytorch compute the gradients for a simple linear regression model?

Input:
I am using pytorch and trying to understand how a simple linear regression model works.

I'm using a simple LinearRegressionModel class:

class LinearRegressionModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LinearRegressionModel, self).__init__()
        self.linear = nn.Linear(input_dim, output_dim)

    def forward(self, x):
        out = self.linear(x)
        return out

model = LinearRegressionModel(1, 1)

Next I instantiate a loss criterion and an optimizer

criterion = nn.MSELoss()

optimizer = torch.optim.SGD(model.parameters(), lr=0.01)

Finally to train the model I use the following code:

for epoch in range(epochs):
    if torch.cuda.is_available():
        inputs = Variable(torch.from_numpy(x_train).cuda())

    if torch.cuda.is_available():
        labels = Variable(torch.from_numpy(y_train).cuda())

    # Clear gradients w.r.t. parameters
    optimizer.zero_grad()

    # Forward to get output
    outputs = model(inputs)

    # Calculate Loss
    loss = criterion(outputs, labels)

    # Getting gradients w.r.t. parameters
    loss.backward()

    # Updating parameters
    optimizer.step()

My question is how does the optimizer get the loss gradient, computed by loss.backward(), to update the parameters using the step() method? How are the model, the loss criterion and the optimizer tied together?

Output:
PyTorch has this concept of tensors and variables. When you use nn.Linear the function creates 2 variables namely W and b.In pytorch a variable is a wrapper that encapsulates a tensor , its gradient and information about its create function. you can directly access the gradients by

w.grad

When you try it before calling the loss.backward() you get None. Once you call the loss.backward() it will contain now gradients. Now you can update these gradient manually with the below simple steps.

w.data -= learning_rate * w.grad.data

When you have a complex network ,the above simple step could grow complex. So optimisers like SGD , Adam takes care of this. When you create the object for these optimisers we pass in the parameters of our model. nn.Module contains this parameters() function which will return all the learnable parameters to the optimiser. Which can be done using the below step.

optimizer = torch.optim.SGD(model.parameters(), lr=0.01)


Instruction: Static graph is fast. Dynamic graph is slow. Is there any specific benchmark demonstrating this?

Input:
I've see some benchmark about tensorflow and pytorch. Tensorflow maybe faster but seems not that faster even sometimes slower.

Is there any benchmark on specifically testing on static graph and dynamic graph demonstrating that static graph is much faster that dynamic graph?

Output:
To be more precise, the speed benefit comes from "deferred execution with graph rewriting."

It's typically associated with explicit graph frameworks (Theano/TF), but with enough engineering you could add it to execution models like numpy/PyTorch which don't have explicit graph. See Bohrium for an example of hacking numpy to do rewriting.

Note that presence of this feature makes the framework less friendly for prototyping, so if you add this to PyTorch, you'll get the same problems that people complain about in TensorFlow

Deferred execution means exceptions can be triggered much later, not when you entered the problematic line
Rewriting means the errors can now be thrown in nodes you didn't create, which gives uninformative error messages

As far as performance, here's a toy benchmark in TensorFlow showing 5x speed-up when you turn on graph rewriting.

I crafted the example to be bottlenecked by memory bandwidth, so it's a no-brainer that graph rewriting (cwise fusion), will give significant speed-boost there. For production LSTM model Google reported 1.8 speed-up when turning on graph optimizations (through XLA)


Instruction: When to use individual optimizers in PyTorch?

Input: The example given here uses two optimizers for encoder and decoder individually. Why? And when to do like that?

Output:
If you have multiple networks (in the sense of multiple objects that inherit from nn.Module), you have to do this for a simple reason: When construction a torch.nn.optim.Optimizer object, it takes the parameters which should be optimized as an argument. In your case:

encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)

This also gives you the freedom to vary parameters as the learning rate independently. If you don't need that, you could create a new class inheriting from nn.Module and containing both networks, encoder and decoder or create a set of parameters to give to the optimizer as explained here:

nets = [encoder, decoder]
parameters = set()
for net in nets:
    parameters |= set(net.parameters())

where | is the union operator for sets in this context.


Instruction: n*m*3 input image to an nxm label in PyTorch

Input:
The input to my network is an RGB image with dimensions nm, how can I get the output to have dimensions of nm.

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 20, kernel_size = 5)
        self.conv2 = nn.Conv2d(20, 50, kernel_size = 3)
        self.conv3 = nn.ConvTranspose2d(50,20, kernel_size = 5)
        self.conv4 = nn.ConvTranspose2d(20,1, kernel_size = 3)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = F.relu(self.conv3(x))
        x = F.relu(self.conv4(x))
    return x

I currently output a 1 * n * m. How can I output an n*m?

Output:
If you want to reshape a Tensor into a different size but with the same number of elements, generally you can use torch.view.

For your case, there is an even simpler solution: torch.squeeze returns a Tensor with all dimensions of size 1 removed.


Instruction: CUDNN error using pretrained vgg16 model

Input:
I am trying to extract the activations of the last layer in a VGG16 model.
For that end I used a decorator over the model as shown below.

When I pass a cuda tensor to the model I get a CUDNN_STATUS_INTERNAL_ERROR with the following traceback.

Anyone knows where I went wrong?

traceback:

  File "/media/data1/iftachg/frame_glimpses/parse_files_to_vgg.py", line 80, in get_activation
    return model(image)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/media/data1/iftachg/frame_glimpses/partial_vgg.py", line 24, in forward
    x = self.vgg16.features(x)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/modules/container.py", line 64, in forward
    input = module(input)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/modules/module.py", line 206, in __call__
    result = self.forward(*input, **kwargs)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/modules/conv.py", line 237, in forward
    self.padding, self.dilation, self.groups)
  File "/media/data1/iftachg/miniconda2/lib/python2.7/site-packages/torch/nn/functional.py", line 39, in conv2d
    return f(input, weight, bias)
RuntimeError: CUDNN_STATUS_INTERNAL_ERROR

Class:

class partial_vgg(nn.Module):

    def __init__(self):
        super(partial_vgg, self).__init__()
        self.vgg16 = models.vgg16(pretrained=True).cuda()
        for param in self.vgg16.parameters():
            param.requires_grad = False

    def forward(self, x):

        x = self.vgg16.features(x)
        x = x.view(x.size(0), -1)
        for l in list(self.vgg16.classifier.children())[:-3]:
            x = l(x)
        return x

Output: Apparently cudnn errors are extremely unhelpful and there was no problem with the code itself - it is simply the GPUs I was trying to access were already in use.


Instruction: Running a portion of Python code in parallel on two different GPUs

Input:
I have a PyTorch script similar to the following:

# Loading data
train_loader, test_loader = someDataLoaderFunction()

# Define the architecture
model = ResNet18()
model = model.cuda()

# Get method from program argument
method = args.method

# Training
train(method, model, train_loader, test_loader)

In order to run the script with two different methods (method1 and method2), it suffices to run the following commands in two different terminals:

CUDA_VISIBLE_DEVICES=0 python program.py --method method1
CUDA_VISIBLE_DEVICES=1 python program.py --method method2

The problem is, the above data loader function contains some randomness in it, which means that the two methods were applied to two different sets of training data. I would like them to train the exact same set of data, so I modified the script as follows:

# Loading data
train_loader, test_loader = someDataLoaderFunction()

# Define the architecture
model = ResNet18()
model = model.cuda()

## Run for the first method
method = 'method1'

 # Training
train(method, model, train_loader, test_loader)

## Run for the second method
method = 'method2'

# Must re-initialize the network first
model = ResNet18()
model = model.cuda()

 # Training
train(method, model, train_loader, test_loader)

Is it possible to make it run in parallel for each method?
Thank you so much in advance for your help!

Output:
I guess the easiest way would be to fix the seeds as below.

myseed=args.seed
np.random.seed(myseed)
torch.manual_seed(myseed)
torch.cuda.manual_seed(myseed)

This should force the data loaders to get the same samples every time. The parallel way is to use multithreading but I hardly see it worth the hassle for the problem you posted.


Instruction: NumPy/PyTorch extract subsets of images

Input:
In Numpy, given a stack of large images A of size(N,hl,wl), and coordinates x of size(N) and y of size(N) I want to get smaller images of size (N,16,16)

In a for loop it would look like this:

B=numpy.zeros((N,16,16))
for i in range(0,N):
  B[i,:,:]=A[i,y[i]:y[i]+16,x[i]:x[i]+16]

But can I do this just with indexing?
Bonus question: Will this indexing also work in pytorch? If not how can I implement this there?

Output:
Pretty simple really with view_as_windows from scikit-image, to get those sliding windowed views as a 6D array with the fourth axis being singleton. Then, use advanced-indexing to select the ones we want based off the y and x indices for indexing into the second and third axes of the windowed array to get our B.

Hence, the implementation would be -

from skimage.util.shape import view_as_windows

BSZ = 16, 16 # Blocksize
A6D = view_as_windows(A,(1,BSZ[0],BSZ[1]))
B_out = A6D[np.arange(N),y,x,0]

Explanation

To explain to other readers on what's really going on with the problem, here's a sample run on a smaller dataset and with a blocksize of (2,2) -

1) Input array (3D) :

In [78]: A
Out[78]:
array([[[ 5,  5,  3,  5,  3,  8],
        [ 5,  *2,  6,  2,  2,  4],
        [ 4,  3,  4,  9,  3,  8],
        [ 6,  3,  3, 10,  4,  5],
        [10,  2,  5,  7,  6,  7],
        [ 5,  4,  2,  5,  2, 10]],

       [[ 4,  9,  8,  4,  9,  8],
        [ 7, 10,  8,  2, 10,  9],
        [10,  *9,  3,  2,  4,  7],
        [ 5, 10,  8,  3,  5,  4],
        [ 6,  8,  2,  4, 10,  4],
        [ 2,  8,  6,  2,  7,  5]],

       [[ *4,  8,  7,  2,  9,  9],
        [ 2, 10,  2,  3,  8,  8],
        [10,  7,  5,  8,  2, 10],
        [ 7,  4, 10,  9,  6,  9],
        [ 3,  4,  9,  9, 10,  3],
        [ 6,  4, 10,  2,  6,  3]]])

2) y and x indices to index into the second and third axes :

In [79]: y
Out[79]: array([1, 2, 0])

In [80]: x
Out[80]: array([1, 1, 0])

3) Finally the desired output, which is a block each from each of the 2D slice along the first axis and whose starting point (top left corner point) is (y,x) on that 2D slice. Refer to the asterisks in A for those  -

In [81]: B
Out[81]:
array([[[ 2,  6],
        [ 3,  4]],

       [[ 9,  3],
        [10,  8]],

       [[ 4,  8],
        [ 2, 10]]])


Instruction: error occurs when loading data in pytorch: 'Image' object has no attribute 'shape'

Input:
I am finetuning resnet152 using the code based on ImageNet training in PyTorch, and error occurs when I load the data, and it occured only after handling several batches of images. How can I solve the problem.
Following code is  the simplified code that produces the same error :

code

# Data loading code
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])

train_loader = torch.utils.data.DataLoader(
    datasets.ImageFolder(train_img_dir, transforms.Compose([
        transforms.RandomSizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        normalize,
    ])),
    batch_size=256, shuffle=True,
    num_workers=1, pin_memory=True)
for i, (input_x, target) in enumerate(train_loader):
    if i % 10 == 0:
        print(i)
        print(input_x.shape)
        print(target.shape)

error

0
torch.Size([256, 3, 224, 224])
torch.Size([256])
10
torch.Size([256, 3, 224, 224])
torch.Size([256])
20
torch.Size([256, 3, 224, 224])
torch.Size([256])
30
torch.Size([256, 3, 224, 224])
torch.Size([256])
----------------------------------------------------------------------
AttributeError                       Traceback (most recent call last)
<ipython-input-48-792d6ca206df> in <module>()
----> 1 for i, (input_x, target) in enumerate(train_loader):
      2     if i % 10 == 0:
      3 #     sample_img = input_x[0]
      4         print(i)
      5         print(input_x.shape)

/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py in __next__(self)
    200                 self.reorder_dict[idx] = batch
    201                 continue
--> 202             return self._process_next_batch(batch)
    203
    204     next = __next__  # Python 2 compatibility

/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py in _process_next_batch(self, batch)
    220         self._put_indices()
    221         if isinstance(batch, ExceptionWrapper):
--> 222             raise batch.exc_type(batch.exc_msg)
    223         return batch
    224

AttributeError: Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 41, in _worker_loop
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 41, in <listcomp>
    samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/datasets/folder.py", line 118, in __getitem__
    img = self.transform(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 369, in __call__
    img = t(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 706, in __call__
    i, j, h, w = self.get_params(img)
  File "/usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py", line 693, in get_params
    w = min(img.size[0], img.shape[1])
AttributeError: 'Image' object has no attribute 'shape'

Output:
There's a bug in transforms.RandomSizedCrop.get_params(). In the last line of your error message, it should be img.size instead of img.shape.

The lines containing the bug will only be executed if the cropping failed for 10 consecutive times (where it falls back to central crop). That's why this error does not occur for every batch of images.

I've submitted a PR to fix it. For a quick fix, you can edit your /usr/local/lib/python3.5/dist-packages/torchvision-0.1.9-py3.5.egg/torchvision/transforms.py file and change all img.shape to img.size.

EDIT: the PR is merged. You can install the latest torchvision on GitHub to fix it.


Instruction: Using Augmented Data Images in Testing

Input:
I am working on a Person Re-Identification problem and am showing the results using a CMC curve.
I used augmented data/Image along with the normal images (Currently training on CUHK01) in the training set. While testing if I don't use the augmented data along with my normal test images for Calculating Rank let's say Rank_1 I get Rank_1 of ~30% on the other hand on using augmented data gives me a Rank_1 of ~65-70% (which is weirdly high regarding the current Rank_1 accuracy in the world).

So my questions are

a) How does augmented data affect the testing set especially in my case.

b) Am I over-fitting or something of that sort.

c) Is it a general rule to avoid usage of augmented images in the test case.

Output:
The reason behind using data augmentation is to reduce the chance of overfitting. This way you want to tell your model that the parameters (theta) are not correlated with the data that you are augmenting (alpha). That is achievable by augmenting each input by every possible alpha. But this is far from reality for a number of reasons, e.g. time/memory limitation, you might not be able to construct every possible augmentation, etc. so there might be some bias. Nevertheless, it still reduces the chance of overfitting to your dataset, but it might overfit to your augmentation.

Thus, if you have the augmentation, you might get more accuracy by matching to the augmented data due to the overfitting, which is an answer to question a. So I believe that the answer to the question b is yes.

In order to answer question c, I have not read about rules for data augmentation but in the literature of machine learning I presume that they avoid any augmentation on the test set. e.g. I quote from a paper

  We augment the training images by
  replacing the green screen with random background images, and
  vary the appearance in terms of color and shading by intrinsic recoloring


Instruction: pytorch backprop through volatile variable error

Input:
I'm trying to minimize some input relative to some target by running it through several backward pass iterations and updating the input at each step. The first pass runs successfully but I get the following error on the second pass:

RuntimeError: element 0 of variables tuple is volatile

This code snippet demonstrates the problem

import torch
from torch.autograd import Variable
import torch.nn as nn

inp = Variable(torch.Tensor([1]), requires_grad=True)
target = Variable(torch.Tensor([3]))

loss_fn = nn.MSELoss()

for i in range(2):
    loss = loss_fn(inp, target)
    loss.backward()
    gradient = inp.grad
    inp = inp - inp.grad * 0.01

When I inspect the value of inp, before it is reassigned on the last line, inp.volatile => False and inp.requires_grad => True but after it is reassigned those switch to True and False, respectively. Why does being a volatile variable prevent the second backprop run?

Output:
You must zero out the gradient before each update like this:

inp.grad.data.zero_()

But in your code every time you update the gradient you are creating another Variable object, so you must update entire history like this:

import torch
from torch.autograd import Variable
import torch.nn as nn

inp_hist = []
inp = Variable(torch.Tensor([1]), requires_grad=True)
target = Variable(torch.Tensor([3]))

loss_fn = nn.MSELoss()

for i in range(2):
    loss = loss_fn(inp, target)
    loss.backward()
    gradient = inp.grad
    inp_hist.append(inp)
    inp = inp - inp.grad * 0.01
    for inp in inp_hist:
        inp.grad.data.zero_()

But this way you will compute the gradient for all previous inputs you have created in the history(and it's bad, it's a wast of everything), a correct implementation looks like this:

import torch
from torch.autograd import Variable
import torch.nn as nn
inp = Variable(torch.Tensor([1]), requires_grad=True)
target = Variable(torch.Tensor([3]))
loss_fn = nn.MSELoss()
for i in range(2):
    loss = loss_fn(inp, target)
    loss.backward()
    gradient = inp.grad
    inp.data = inp.data - inp.grad.data * 0.01
    inp.grad.data.zero_()


Instruction: How to get ouput from a particular layer from pretrained CNN in pytorch

Input:
I have pretrained CNN (RESNET18) on imagenet dataset , now what i want is to get output of my input image from a particular layer,

for example.

my input image is of FloatTensor(3, 224, 336) and i send a batch of size = 10 in my resnet model , now what i want is the output returned by model.layer4,

Now what i tried is out = model.layer4(Variable(input)) but it gave me input dimensions mismatch error(As expected), this is the exact error returned

RuntimeError: Need input of dimension 4 and input.size[1] == 64 but got input to be of shape: [10 x 3 x 224 x 336] at /Users/soumith/miniconda2/conda-bld/pytorch_1501999754274/work/torch/lib/THNN/generic/SpatialConvolutionMM.c:47

So i'm confused , how to proceed now to get my layer4 output

PS: My ultimate task is to combine layer4 output and fullyconnected layer output together (Tweeking in CNN, kind of gated CNN ), so if anyone have any insight in this case  then please do tell me, maybe my above approach is not right

Output:
You must create a module with all layers from start to the block you want:

resnet = torchvision.models.resnet18(pretrained=True)
f = torch.nn.Sequential(*list(resnet.children())[:6])
features = f(imgs)


Instruction: How to use groups parameter in PyTorch conv2d function

Input:
I am trying to compute a per-channel gradient image in PyTorch. To do this, I want to perform a standard 2D convolution with a Sobel filter on each channel of an image. I am using the torch.nn.functional.conv2d function for this

In my minimum working example code below, I get an error:

import torch
import torch.nn.functional as F

filters = torch.autograd.Variable(torch.randn(1,1,3,3))
inputs = torch.autograd.Variable(torch.randn(1,3,10,10))
out = F.conv2d(inputs, filters, padding=1)

  RuntimeError: Given groups=1, weight[1, 1, 3, 3], so expected
  input[1, 3, 10, 10]  to have 1 channels, but got 3 channels instead

This suggests that groups need to be 3. However, when I make groups=3, I get a different error:

import torch
import torch.nn.functional as F

filters = torch.autograd.Variable(torch.randn(1,1,3,3))
inputs = torch.autograd.Variable(torch.randn(1,3,10,10))
out = F.conv2d(inputs, filters, padding=1, groups=3)

  RuntimeError: invalid argument 4: out of range at
  /usr/local/src/pytorch/torch/lib/TH/generic/THTensor.c:440

When I check that code snippet in the THTensor class, it refers to a bunch of dimension checks, but I don't know where I'm going wrong.

What does this error mean? How can I perform my intended convolution with this conv2d function? I believe I am misunderstanding the groups parameter.

Output:
If you want to apply a per-channel convolution then your out-channel should be the same as your in-channel. This is expected, considering each of your input channels creates a separate output channel that it corresponds to.

In short, this will work

import torch
import torch.nn.functional as F

filters = torch.autograd.Variable(torch.randn(3,1,3,3))
inputs = torch.autograd.Variable(torch.randn(1,3,10,10))
out = F.conv2d(inputs, filters, padding=1, groups=3)

whereas, filters of size (2, 1, 3, 3) or (1, 1, 3, 3) will not work.

Additionally, you can also make your out-channel a multiple of in-channel. This works for instances where you want to have multiple convolution filters for each input channel.

However, This only makes sense if it is a multiple. If not, then pytorch falls back to its closest multiple, a number less than what you specified. This is once again expected behavior. For example a filter of size (4, 1, 3, 3) or (5, 1, 3, 3), will result in an out-channel of size 3.


Instruction: expected Double tensor (got Float tensor) in Pytorch

Input:
I want to create nn.Module in Pytorch. I used following code for text related problem (In fact I use Glove 300d pre-trained embedding and weighted average of words in a sentence to do classification).

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv1d(300, 128, kernel_size=5)
        self.conv2 = nn.Conv1d(128, 64, kernel_size=2)
        self.conv2_drop = nn.Dropout()
        self.fc1 = nn.Linear(64, 20)
        self.fc2 = nn.Linear(20, 2)

    def forward(self, x):
        x = F.relu(F.avg_pool1d(self.conv1(x), 2))
        x = F.relu(F.avg_pool1d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(-1, 1)
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        return self.fc2(x)

But it gives me following error:

Traceback (most recent call last):
    x = F.relu(F.avg_pool1d(self.conv1(x), 2))
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/modules/module.py", line 224, in __call__
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/modules/conv.py", line 154, in forward
    self.padding, self.dilation, self.groups)
  File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/torch/nn/functional.py", line 83, in conv1d
    return f(input, weight, bias)
RuntimeError: expected Double tensor (got Float tensor)

I fairly new to Conv1d, and most of tutorial used Conv1d for image problems. Can anybody give me some idea what's the problem?

I also added model.double() inside forward method but gives me another error:

RuntimeError: Given input size: (300 x 1 x 1). Calculated output size: (128 x 1 x -3). Output size is too small

Output:
Error 1

  RuntimeError: expected Double tensor (got Float tensor)

This happens when you pass a double tensor to the first conv1d function. Conv1d only works with float tensor.
Either do,

conv1.double() , or
model.double().

Which is what you have done, and that is correct.

Error 2

  RuntimeError: Given input size: (300 x 1 x 1). Calculated output size: (128 x 1 x -3). Output size is too small

This is because you are passing an input to which a convolution with a window size of 5 is not valid. You will have to add padding to your Conv1ds for this to work, like so:

self.conv1 = nn.Conv1d(300, 128, kernel_size=5, padding=2)

If you dont wish to add padding, then given (batch_size, in_channels, inp_size) as the size of the input tensor, you have to make sure that your inp_size is greater than 5.

 All fixes combined

Make sure that your sizes are correct for the rest of your network. Like so:

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv1d(300, 128, kernel_size=5, padding=2)
        self.conv2 = nn.Conv1d(128, 64, kernel_size=2, padding=1)
        self.conv2_drop = nn.Dropout()
        self.fc1 = nn.Linear(64, 20)
        self.fc2 = nn.Linear(20, 2)

    def forward(self, x):
        x = F.relu(F.avg_pool1d(self.conv1(x), 2, padding=1))
        x = F.relu(F.avg_pool1d(self.conv2_drop(self.conv2(x)), 2))
        x = x.view(1, -1) # bonus fix, Linear needs (batch_size, in_features) and not (in_features, batch_size) as input.
        x = F.relu(self.fc1(x))
        x = F.dropout(x, training=self.training)
        return self.fc2(x)

if __name__ == '__main__':

    t = Variable(torch.randn((1, 300, 1))).double() # t is a double tensor
    model = Net()
    model.double() # this will make sure that conv1d will process double tensor
    out = model(t)


Instruction: Jupyter Kernel crash/dies when use large Neural Network layer, any idea pls?

Input:
I am experimenting Autoencoder with Pytorch. It seems when I use relatively larger neural network for instance nn.Linear(250*250, 40*40) as the first layer, the Jupyter kernel keep crashing. when I use smaller layer size e.g. nn.Linear(250*250, 20*20). the Jupyter kernel is ok. Any idea how to fix this? So I can run larger network.Thank you. The entire network is as below.

# model:
class AutoEncoder(nn.Module):
    def __init__(self):
        super().__init__()

        self.encoder = nn.Sequential(
            nn.Linear(250*250, 20*20),
            nn.BatchNorm1d(20*20,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(20*20, 20*20),
            nn.BatchNorm1d(20*20,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(20*20, 20*20),
            nn.BatchNorm1d(20*20,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(20*20, 15*15),
            nn.BatchNorm1d(15*15,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(15*15, 3),
            nn.BatchNorm1d(3,momentum=0.5),
            #nn.Dropout(0.5),
            #nn.Tanh(),
            #nn.Linear(5*5,5),
        )
        self.decoder = nn.Sequential(
            #nn.Linear(5, 5*5),
            #nn.BatchNorm1d(5*5,momentum=0.5),
            #nn.Dropout(0.5),
            #nn.Tanh(),
            nn.Linear(3, 15*15),
            nn.BatchNorm1d(15*15,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(15*15, 20*20),
            nn.BatchNorm1d(20*20,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(20*20, 20*20),
            nn.BatchNorm1d(20*20,momentum=0.5),
            nn.Dropout(0.5),
            nn.LeakyReLU(),
            nn.Linear(20*20, 250*250),
            nn.BatchNorm1d(250*250,momentum=0.5),
            nn.Dropout(0.5),
            nn.Sigmoid(),
        )
    def forward(self, x):
        encoded = self.encoder(x)
        decoded = self.decoder(encoded)
        return encoded, decoded

Output: I have found the root cause. I am running a docker ubuntu image/package on windows. the memory setting is set too low, when I increase the memory setting on docker. my ubuntu environment got more memory, then I can larger matrix operations.


Instruction: PyTorch Broadcasting Failing. "Rules" Followed

Input:
I have a few PyTorch examples that confuse me and am hoping to clear it up.

Firstly, as per the PyTorch page, I would expect these example to work as do their numpy equivalents namely these. The first example is very intuitive. These are compatible for broadcasting:

Image  (3d array): 256 x 256 x 3
Scale  (1d array):             3
Result (3d array): 256 x 256 x 3

Take for instance these:

torch.Tensor([[1,2,3]])/torch.Tensor([1,2,3])
Out[5]:

 1  1  1
[torch.FloatTensor of size 1x3]
torch.Tensor([[1,2,3]])/torch.Tensor([1,2,3])
Out[6]:

  1  1  1
[torch.FloatTensor of size 1x3]

torch.Tensor([[[1,2,3]]])/torch.Tensor([1,2,3])
Out[7]:

(0 ,.,.) =
  1  1  1
[torch.FloatTensor of size 1x1x3]

However, this is the result of the numpy example:

torch.randn(256,256,3)/torch.Tensor([1,2,3])
Traceback (most recent call last):

  File "<ipython-input-12-4c328d453e24>", line 1, in <module>
    torch.randn(256,256,3)/torch.Tensor([1,2,3])

  File "/home/nick/anaconda3/lib/python3.6/site-packages/torch/tensor.py", line 313, in __div__
    return self.div(other)

RuntimeError: inconsistent tensor size at /opt/conda/conda-bld/pytorch_1501971235237/work/pytorch-0.1.12/torch/lib/TH/generic/THTensorMath.c:873

Here is an excerpt which says this ought to work:

  Two tensors are “broadcastable” if the following rules hold:

  Each tensor has at least one dimension.
  When iterating over the dimension sizes, starting at the trailing dimension, the dimension sizes must either be equal, one of them is 1, or one of them does not exist.

If one converts the tensors to numpy arrays the arithmetic works as expected.

What's going wrong? Have I misinterpreted the docs and if so what syntax results in the same results?

Output:
Verify that you are using the correct version of pytorch. It must be 0.2.0, which is when broadcasting was introduced in pytorch.

In[2]: import torch
In[3]: torch.__version__

Out[3]: '0.2.0_4'


Instruction: Porting PyTorch code from CPU to GPU

Input:
Following the tutorial from https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation.ipynb

There is a USE_CUDA flag that is used to control the variable and tensor types between CPU (when False) to GPU (when True) types.

Using the data from en-fr.tsv and converting the sentences to variables:

import unicodedata
import string
import re
import random
import time
import math

from gensim.corpora.dictionary import Dictionary

import torch
import torch.nn as nn
from torch.autograd import Variable
from torch import LongTensor, FloatTensor
from torch import optim
import torch.nn.functional as F

import numpy as np

MAX_LENGTH = 10
USE_CUDA = False

# Turn a Unicode string to plain ASCII, thanks to http://stackoverflow.com/a/518232/2809427
def unicode_to_ascii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
    )

# Lowercase, trim, and remove non-letter characters
def normalize_string(s):
    s = unicode_to_ascii(s.lower().strip())
    s = re.sub(r"([.!?])", r" \1", s)
    s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
    return s

SOS_IDX, SOS_TOKEN = 0, '<s>'
EOS_IDX, EOS_TOKEN = 1, '</s>'
UNK_IDX, UNK_TOKEN = 2, '<unk>'
PAD_IDX, PAD_TOKEN = 3, '<blank>'

lines = open('en-fr.tsv').read().strip().split('\n')
pairs = [[normalize_string(s).split() for s in l.split('\t')] for l in lines]
src_sents, trg_sents = zip(*pairs)

src_dict = Dictionary([[SOS_TOKEN, EOS_TOKEN, UNK_TOKEN, PAD_TOKEN]])
src_dict.add_documents(src_sents)

trg_dict = Dictionary([[SOS_TOKEN, EOS_TOKEN, UNK_TOKEN, PAD_TOKEN]])
trg_dict.add_documents(trg_sents)

def variablize_sentences(sentence, dictionary):
    indices = [dictionary.token2id[tok] for tok in sentence] + [dictionary.token2id[EOS_TOKEN]]
    var = Variable(LongTensor(indices).view(-1, 1))
    return var.cuda() if USE_CUDA else var

input_variables = [variablize_sentences(sent, src_dict) for sent in src_sents]
output_variables = [variablize_sentences(sent, trg_dict) for sent in trg_sents]

And using a Encoder-Attn-Decoder network:

class EncoderRNN(nn.Module):
    def __init__(self, input_size, hidden_size, n_layers=1):
        super(EncoderRNN, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers

        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)

        self.embedding = self.embedding.cuda() if USE_CUDA else self.embedding
        self.gru = self.gru.cuda() if USE_CUDA else self.gru

    def forward(self, word_inputs, hidden):
        seq_len = len(word_inputs)

        embedded = self.embedding(word_inputs).view(seq_len, 1, -1)
        embedded = embedded.cuda() if USE_CUDA else embedded

        output, hidden = self.gru(embedded, hidden)
        output = output.cuda() if USE_CUDA else output
        hiddne = hidden.cuda() if USE_CUDA else hidden

        return output, hidden

    def init_hidden(self):
        hidden = Variable(torch.zeros(self.n_layers, 1, self.hidden_size))
        return hidden.cuda() if USE_CUDA else hidden

class Attn(nn.Module):
    def __init__(self, method, hidden_size, max_length=MAX_LENGTH):
        super(Attn, self).__init__()

        self.method = method
        self.hidden_size = hidden_size

        if self.method == 'general':
            self.attn = nn.Linear(self.hidden_size, hidden_size)

        elif self.method == 'concat':
            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)
            self.other = nn.Parameter(FloatTensor(1, hidden_size))

    def forward(self, hidden, encoder_outputs):
        seq_len = len(encoder_outputs)

        # Create variable to store attention energies
        attn_energies = Variable(torch.zeros(seq_len)) # B x 1 x S
        attn_energies = attn_energies.cuda() if USE_CUDA else attn_energies
        # Calculate energies for each encoder output
        for i in range(seq_len):
            attn_energies[i] = self.score(hidden, encoder_outputs[i])

        # Normalize energies to weights in range 0 to 1, resize to 1 x 1 x seq_len
        return F.softmax(attn_energies).unsqueeze(0).unsqueeze(0)

    def score(self, hidden, encoder_output):
        if self.method == 'dot':
            energy =torch.dot(hidden.view(-1), encoder_output.view(-1))
        elif self.method == 'general':
            energy = self.attn(encoder_output)
            energy = torch.dot(hidden.view(-1), energy.view(-1))
        elif self.method == 'concat':
            energy = self.attn(torch.cat((hidden, encoder_output), 1))
            energy = torch.dot(self.v.view(-1), energy.view(-1))
        return energy

class AttnDecoderRNN(nn.Module):
    def __init__(self, attn_model, hidden_size, output_size, n_layers=1, dropout_p=0.1):
        super(AttnDecoderRNN, self).__init__()

        # Keep parameters for reference
        self.attn_model = attn_model
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers
        self.dropout_p = dropout_p

        # Define layers
        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size * 2, hidden_size, n_layers, dropout=dropout_p)
        self.out = nn.Linear(hidden_size * 2, output_size)

        self.embedding = self.embedding.cuda() if USE_CUDA else self.embedding
        self.gru = self.gru.cuda() if USE_CUDA else self.gru
        self.out = self.out.cuda() if USE_CUDA else self.out

        # Choose attention model
        if attn_model != 'none':
            self.attn = Attn(attn_model, hidden_size)
            self.attn = self.attn.cuda() if USE_CUDA else self.attn

    def forward(self, word_input, last_context, last_hidden, encoder_outputs):
        # Note: we run this one step at a time

        # Get the embedding of the current input word (last output word)
        word_embedded = self.embedding(word_input).view(1, 1, -1) # S=1 x B x N

        # Combine embedded input word and last context, run through RNN
        rnn_input = torch.cat((word_embedded, last_context.unsqueeze(0)), 2)
        rnn_output, hidden = self.gru(rnn_input, last_hidden)

        # Calculate attention from current RNN state and all encoder outputs; apply to encoder outputs
        attn_weights = self.attn(rnn_output.squeeze(0), encoder_outputs)
        context = attn_weights.bmm(encoder_outputs.transpose(0, 1)) # B x 1 x N

        # Final output layer (next word prediction) using the RNN hidden state and context vector
        rnn_output = rnn_output.squeeze(0) # S=1 x B x N -> B x N
        context = context.squeeze(1)       # B x S=1 x N -> B x N
        output = F.log_softmax(self.out(torch.cat((rnn_output, context), 1)))

        if USE_CUDA:
            return output.cuda(), context.cuda(), hidden.cuda(), attn_weights.cuda()
        else:
            return output, context, hidden, attn_weights

And testing the network:

encoder_test = EncoderRNN(10, 10, 2) # I, H , L
decoder_test = AttnDecoderRNN('general', 10, 10, 2) # A, H, O, L

encoder_hidden = encoder_test.init_hidden()
if USE_CUDA:
    word_inputs = Variable(torch.LongTensor([1, 2, 3]).cuda())
else:
    word_inputs = Variable(torch.LongTensor([1, 2, 3]))
encoder_outputs, encoder_hidden = encoder_test(word_inputs, encoder_hidden)
decoder_attns = torch.zeros(1, 3, 3)
decoder_hidden = encoder_hidden
decoder_context = Variable(torch.zeros(1, decoder_test.hidden_size))

decoder_output, decoder_context, decoder_hidden, decoder_attn = decoder_test(word_inputs[0], decoder_context, decoder_hidden, encoder_outputs)
print(decoder_output)
print(decoder_hidden)
print(decoder_attn)

The code works fine on CPU,

[out]:

EncoderRNN (
  (embedding): Embedding(10, 10)
  (gru): GRU(10, 10, num_layers=2)
)
AttnDecoderRNN (
  (embedding): Embedding(10, 10)
  (gru): GRU(20, 10, num_layers=2, dropout=0.1)
  (out): Linear (20 -> 10)
  (attn): Attn (
    (attn): Linear (10 -> 10)
  )
)
Variable containing:
-2.4378 -2.3556 -2.3391 -2.5070 -2.3439 -2.3415 -2.3976 -2.1832 -1.9976 -2.2213
[torch.FloatTensor of size 1x10]

Variable containing:
(0 ,.,.) =

Columns 0 to 8
  -0.2325  0.0775  0.5415  0.4876 -0.5771 -0.0687  0.1832 -0.5285  0.2508

Columns 9 to 9
  -0.1837

(1 ,.,.) =

Columns 0 to 8
  -0.1389 -0.2605 -0.0518  0.3405  0.0774  0.1815  0.0297 -0.1304 -0.1015

Columns 9 to 9
   0.2602
[torch.FloatTensor of size 2x1x10]

Variable containing:
(0 ,.,.) =
  0.3334  0.3291  0.3374
[torch.FloatTensor of size 1x1x3]

but when changing the flag to USE_GPU=True, it throws the error when initializing the decoder_test object, it throws a TypeError:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-76-b3c660013934> in <module>()
     12 decoder_context = Variable(torch.zeros(1, decoder_test.hidden_size))
     13
---> 14 decoder_output, decoder_context, decoder_hidden, decoder_attn = decoder_test(word_inputs[0], decoder_context, decoder_hidden, encoder_outputs)
     15 print(decoder_output)
     16 print(decoder_hidden)

~/.local/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    222         for hook in self._forward_pre_hooks.values():
    223             hook(self, input)
--> 224         result = self.forward(*input, **kwargs)
    225         for hook in self._forward_hooks.values():
    226             hook_result = hook(self, input, result)

<ipython-input-75-34ecfe9b3112> in forward(self, word_input, last_context, last_hidden, encoder_outputs)
     32
     33         # Combine embedded input word and last context, run through RNN
---> 34         rnn_input = torch.cat((word_embedded, last_context.unsqueeze(0)), 2)
     35         rnn_output, hidden = self.gru(rnn_input, last_hidden)
     36

~/.local/lib/python3.5/site-packages/torch/autograd/variable.py in cat(iterable, dim)
    895         @staticmethod
    896         def cat(iterable, dim=0):
--> 897             return Concat.apply(dim, *iterable)
    898
    899         @staticmethod

~/.local/lib/python3.5/site-packages/torch/autograd/_functions/tensor.py in forward(ctx, dim, *inputs)
    315         ctx.dim = dim
    316         ctx.input_sizes = [i.size(dim) for i in inputs]
--> 317         return torch.cat(inputs, dim)
    318
    319     @staticmethod

TypeError: cat received an invalid combination of arguments - got (tuple, int), but expected one of:
 * (sequence[torch.cuda.FloatTensor] seq)
 * (sequence[torch.cuda.FloatTensor] seq, int dim)
      didn't match because some of the arguments have invalid types: (tuple, int)

The question is why are that types not matching in CUDA but it works on CPU and how to resolve this?

Does PyTorch have a global flag to just change all types to CUDA types and not mess around with CPU/GPU types?

Output:
You can also try:

net = YouNetworkClass()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
net.to(device)

After that, you have to send the word_inputs, encoder_hidden and decoder_context to the GPU too:

word_inputs, encoder_hidden, decoder_context = word_inputs.to(device), encoder_hidden.to(device), decoder_context.to(device)

Look here: https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#training-on-gpu


Instruction: RNN is not training (PyTorch)

Input:
I can't get what I am doing wrong when training RNN. I am trying to train RNN for AND operation on sequences (to learn how it works on simple task).
But my network is not learning, loss stays the same and it can't event overfit the model.
Can you please help me to find the problem?

Data I am using:

data = [
    [1, 1, 1, 1, 0, 0, 1, 1, 1],
    [1, 1, 1, 1],
    [0, 0, 1, 1],
    [0, 0, 0, 0, 0, 0, 0],
    [1, 1, 1, 1, 1, 1, 1],
    [1, 1],
    [0],
    [1],
    [1, 0]]
labels = [
    0,
    1,
    0,
    0,
    1,
    1,
    0,
    1,
    0
]

Code for NN:

class AndRNN(nn.Module):
def __init__(self):
    super(AndRNN, self).__init__()
    self.rnn = nn.RNN(1, 10, 5)
    self.fc = nn.Sequential(
        nn.Linear(10, 30),
        nn.Linear(30, 2)
    )

def forward(self, input, hidden):
    x, hidden = self.rnn(input, hidden)
    x = self.fc(x[-1])
    return x, hidden

def initHidden(self):
    return Variable(torch.zeros((5, 1, 10)))

Training loop:

criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

correct = 0
for e in range(20):
for i in range(len(data)):
    tensor = torch.FloatTensor(data[i]).view(-1, 1, 1)
    label = torch.LongTensor([labels[i]])
    hidden = net.initHidden()
    optimizer.zero_grad()

    out, hidden = net(Variable(tensor), Variable(hidden.data))

    _, l = torch.topk(out, 1)
    if label[0] == l[0].data[0]:
        correct += 1

    loss = criterion(out, Variable(label))
    loss.backward()
    optimizer.step()

    print("Loss ", loss.data[0], "Accuracy ", (correct / (i + 1)))

Shape for tensor will be (sequence_len, 1 (which is batch size), 1), that is correct according to the PyTorch docs for RNN

Output:
The problem is this line:

out, hidden = net(Variable(tensor), Variable(hidden.data))

It should be simply

out, hidden = net(Variable(tensor), hidden)

By having Variable(hidden.data) here you are creating a new hidden_state variable (with all zeros) at very step instead of passing the hidden state from the previous state.

I tried your example, and changed the optimizer to Adam. There are the complete code.

class AndRNN(nn.Module):
    def __init__(self):
        super(AndRNN, self).__init__()
        self.rnn = nn.RNN(1, 10, 5)
        self.fc = nn.Sequential(
            nn.Linear(10, 30),
            nn.Linear(30, 2)
        )

    def forward(self, input, hidden):
        x, hidden = self.rnn(input, hidden)
        x = self.fc(x[-1])
        return x, hidden

    def initHidden(self):
        return Variable(torch.zeros((5, 1, 10)))

net = AndRNN()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters())

correct = 0
for e in range(100):
    for i in range(len(data)):
        tensor = torch.FloatTensor(data[i]).view(-1, 1, 1)
        label = torch.LongTensor([labels[i]])
        hidden = net.initHidden()
        optimizer.zero_grad()

        out, hidden = net(Variable(tensor), hidden)

        loss = criterion(out, Variable(label))
        loss.backward()
        optimizer.step()
    if e % 25 == 0:
        print("Loss ", loss.data[0])

Result

Loss  0.6370733976364136
Loss  0.25336754322052
Loss  0.006924811284989119
Loss  0.002351854695007205


Instruction: What does the parameter retain_graph mean in the Variable's backward() method?

Input:
I'm going through the neural transfer pytorch tutorial and am confused about the use of retain_variable(deprecated, now referred to as retain_graph). The code example show:

class ContentLoss(nn.Module):

    def __init__(self, target, weight):
        super(ContentLoss, self).__init__()
        self.target = target.detach() * weight
        self.weight = weight
        self.criterion = nn.MSELoss()

    def forward(self, input):
        self.loss = self.criterion(input * self.weight, self.target)
        self.output = input
        return self.output

    def backward(self, retain_variables=True):
        #Why is retain_variables True??
        self.loss.backward(retain_variables=retain_variables)
        return self.loss

From the documentation

  retain_graph (bool, optional) – If False, the graph used to compute
  the grad will be freed. Note that in nearly all cases setting this
  option to True is not needed and often can be worked around in a much
  more efficient way. Defaults to the value of create_graph.

So by setting retain_graph= True, we're not freeing the memory allocated for the graph on the backward pass. What is the advantage of keeping this memory around, why do we need it?

Output:
@cleros is pretty on the point about the use of retain_graph=True. In essence, it will retain any necessary information to calculate a certain variable, so that we can do backward pass on it.
An illustrative example

Suppose that we have a computation graph shown above. The variable d and e is the output, and a is the input. For example,
import torch
from torch.autograd import Variable
a = Variable(torch.rand(1, 4), requires_grad=True)
b = a**2
c = b*2
d = c.mean()
e = c.sum()

when we do d.backward(), that is fine. After this computation, the parts of the graph that calculate d will be freed by default to save memory. So if we do e.backward(), the error message will pop up. In order to do e.backward(), we have to set the parameter retain_graph to True in d.backward(), i.e.,
d.backward(retain_graph=True)

As long as you use retain_graph=True in your backward method, you can do backward any time you want:
d.backward(retain_graph=True) # fine
e.backward(retain_graph=True) # fine
d.backward() # also fine
e.backward() # error will occur!

More useful discussion can be found here.
A real use case
Right now, a real use case is multi-task learning where you have multiple losses that maybe be at different layers. Suppose that you have 2 losses: loss1 and loss2 and they reside in different layers. In order to backprop the gradient of loss1 and loss2 w.r.t to the learnable weight of your network independently. You have to use retain_graph=True in backward() method in the first back-propagated loss.
# suppose you first back-propagate loss1, then loss2 (you can also do the reverse)
loss1.backward(retain_graph=True)
loss2.backward() # now the graph is freed, and next process of batch gradient descent is ready

optimizer.step() # update the network parameters


Instruction: PyTorch: How to get the shape of a Tensor as a list of int

Input:
In numpy, V.shape gives a tuple of ints of dimensions of V.

In tensorflow V.get_shape().as_list() gives a list of integers of the dimensions of V.

In pytorch, V.size() gives a size object, but how do I convert it to ints?

Output:
For PyTorch v1.0 and possibly above:

>>> import torch
>>> var = torch.tensor([[1,0], [0,1]])

# Using .size function, returns a torch.Size object.
>>> var.size()
torch.Size([2, 2])
>>> type(var.size())
<class 'torch.Size'>

# Similarly, using .shape
>>> var.shape
torch.Size([2, 2])
>>> type(var.shape)
<class 'torch.Size'>

You can cast any torch.Size object to a native Python list:

>>> list(var.size())
[2, 2]
>>> type(list(var.size()))
<class 'list'>

In PyTorch v0.3 and 0.4:

Simply list(var.size()), e.g.:

>>> import torch
>>> from torch.autograd import Variable
>>> from torch import IntTensor
>>> var = Variable(IntTensor([[1,0],[0,1]]))

>>> var
Variable containing:
 1  0
 0  1
[torch.IntTensor of size 2x2]

>>> var.size()
torch.Size([2, 2])

>>> list(var.size())
[2, 2]


Instruction: Pytorch to Keras code equivalence

Input:
Given a below code in PyTorch what would be the Keras equivalent?

class Network(nn.Module):

    def __init__(self, state_size, action_size):
        super(Network, self).__init__()

        # Inputs = 5, Outputs = 3, Hidden = 30
        self.fc1 = nn.Linear(5, 30)
        self.fc2 = nn.Linear(30, 3)

    def forward(self, state):
        x = F.relu(self.fc1(state))
        outputs = self.fc2(x)
        return outputs

Is it this?

model = Sequential()
model.add(Dense(units=30, input_dim=5, activation='relu'))
model.add(Dense(units=30, activation='relu'))
model.add(Dense(units=3, activation='linear'))

or is it this?

model = Sequential()
model.add(Dense(units=30, input_dim=5, activation='linear'))
model.add(Dense(units=30, activation='relu'))
model.add(Dense(units=3, activation='linear'))

or is it?

model = Sequential()
model.add(Dense(units=30, input_dim=5, activation='relu'))
model.add(Dense(units=30, activation='linear'))
model.add(Dense(units=3, activation='linear'))

Thanks

Output:
None of them looks correct according to my knowledge. A correct Keras equivalent code would be:

model = Sequential()
model.add(Dense(30, input_shape=(5,), activation='relu'))
model.add(Dense(3))

  model.add(Dense(30, input_shape=(5,), activation='relu'))

Model will take as input arrays of shape (*, 5) and output arrays of shape (*, 30). Instead of input_shape, you can use input_dim also. input_dim=5 is equivalent to input_shape=(5,).

  model.add(Dense(3))

After the first layer, you don't need to specify the size of the input anymore. Moreover, if you don't specify anything for activation, no activation will be applied (equivalent to linear activation).

Another alternative would be:

model = Sequential()
model.add(Dense(30, input_dim=5))
model.add(Activation('relu'))
model.add(Dense(3))

Hopefully this makes sense!


Instruction: Indexing one PyTorch tensor by another using index_select

Input:
I have a 3 x 3 PyTorch LongTensor that looks something like this:

A =
    [0, 0, 0]
    [1, 2, 2]
    [1, 2, 3]

I want to us it to index a 4 x 2 FloatTensor like this one:

B =
    [0.4, 0.5]
    [1.2, 1.4]
    [0.8, 1.9]
    [2.4, 2.9]

My intended output is the 2 x 3 x 3 FloatTensor below:

C[0,:,:] =
    [0.4, 0.4, 0.4]
    [1.2, 0.8, 0.8]
    [1.2, 0.8, 2.4]

C[1,:,:] =
    [0.5, 0.5, 0.5]
    [1.4, 1.9, 1.9]
    [1.4, 1.9, 2.9]

In other words, matrix A is indexing and broadcasting matrix B. A is the matrix of indices of B, so this operation is essentially an indexing operation.

How can this be done using the torch.index_select() function? If the solution involves adding or permuting dimensions, that's fine.

Output:
Using index_select() requires that the indexing values are in a vector rather than a tensor. But as long as that is formatted correctly, the function handles the broadcasting for you. The last thing that must be done is reshaping the output, I believe due to the broadcasting.

The one-liner that will do this operation successfully is

torch.index_select(B, 0, A.view(-1)).view(3,-1,2).permute(2,0,1)

A.view(-1) vectorizes the indices matrix.

__.view(3,-1,2) reshapes back to the shape of the indexing matrix, but accounting for the new extra dimension of size 2 (as I am indexing an N x 2 matrix).

Lastly, __.permute(2,0,1) reshapes the matrix so that the output views each dimension of B in a separate channel (rather than each column).


Instruction: How to input a matrix to CNN in pytorch

Input:
I'm very new to pytorch and I want to figure out how to input a matrix rather than image into CNN.
I have try it in the following way, but some errors occur.
I define my dataset as following:

class FrameDataSet(tud.Dataset):
    def __init__(self, data):
        targets = data['class'].values.tolist()
        features = data.drop('class', axis=1).astype(np.int64).values

        self.datalist = features.reshape((-1, feature_num, frame_size))
        self.labellist = targets

    def __getitem__(self, index):
        return torch.Tensor(self.datalist[index].astype(float)), self.labellist[index]

    def __len__(self):
        return self.datalist.shape[0]

And my CNN is:

self.conv = nn.Sequential(
        nn.Conv2d(1, 12, 3),
        nn.ReLU(True),
        nn.MaxPool2d(3, 3))
self.fc1 = nn.Linear(80, 100)
self.fc2 = nn.Linear(100, 30)
self.fc3 = nn.Linear(30, 5)

But when the data was inputted to CNN, the error brings:

  File "/home/sparks/anaconda2/lib/python2.7/site-packages/torch/nn/functional.py", line 48, in conv2d
      raise ValueError("Expected 4D tensor as input, got {}D tensor instead.".format(input.dim()))
  Expected 4D tensor as input, got 3D tensor instead.

Output:
Your input probably missing one dimension. It should be:

  (batch_size, channels, width, height)

If you only have one element in the batch, the tensor have to be in your case

  e.g (1, 1, 28, 28)

because your first conv2d-layer expected a 1-channel input.


Instruction: Why detach needs to be called on variable in this example?

Input:
I was going through this example - https://github.com/pytorch/examples/blob/master/dcgan/main.py and I have a basic question.

fake = netG(noise)
label = Variable(label.fill_(fake_label))
output = netD(fake.detach()) # detach to avoid training G on these labels
errD_fake = criterion(output, label)
errD_fake.backward()
D_G_z1 = output.data.mean()
errD = errD_real + errD_fake
optimizerD.step()

I understand that why we call detach() on variable fake, so that no gradients are computed for the Generator parameters. My question is, does it matter since optimizerD.step() is going to update the parameters associated with Discriminator only?

OptimizerD is defined as:
optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))

Besides, in the next step when we will update parameters for Generator, before that we will call netG.zero_grad() which eventually removes all previously computed gradients. Moreover, when we update parameters for G network, we do this - output = netD(fake). Here, we are not using detach. Why?

So, why detaching the variable (line 3) is necessary in the above code?

Output:
ORIGINAL ANSWER (WRONG / INCOMPLETE)

You're right, optimizerD only updates netD and the gradients on netG are not used before netG.zero_grad() is called, so detaching is not necessary, it just saves time, because you're not computing gradients for the generator.

You're basically also answering your other question yourself, you don't detach fake in the second block because you specifically want to compute gradients on netG to be able to update its parameters.

Note how in the second block real_label is used as the corresponding label for fake, so if the discriminator finds the fake input to be real, the final loss is small, and vice versa, which is precisely what you want for the generator. Not sure if that's what confused you, but it's really the only difference compared to training the discriminator on fake inputs.

EDIT

Please see FatPanda's comment! My original answer is in fact incorrect. Pytorch destroys (parts of) the compute graph when .backward() is called. Without detaching before errD_fake.backward() the errG.backward() call later would not be able to backprop into the generator because the required graph is no longer available (unless you specify retain_graph=True). I'm relieved Soumith made the same mistake :D


Instruction: sequence to sequence learning for language translation, what about unseen words

Input:
sequence to sequence learning is a powerful mechanism for language translation, especially using it locally in a context specific case.

I am following this pytorch tutorial for the task.

However, the tutorial did not split the data into training and testing.
You might think its not a big deal, just split it up, use one chunk for training and the other for testing. But it is not that simple.

Essentially, the tutorial creates the indices of the seen words while leading the dataset. The indices are simply stored in the dictionary. This is before going to the encoder RNN, just a simple conversion kind of task from words to the numbers.

If data is split up at random, what happens is, one of the keyword may not appear in the sentences from the training set, and so may not have an index at all. If it shows up at the time of testing, what should be done?

Extend the dictionary?

Output: Sequence to sequence models performance strongly depend on count of unique words in vocabulary. Each unique word has to be encountered a number of times in training set, such that model can learn it correct usage. Words that appears few times cannot be used by the model, as model can't learn enough information about such words. In practice, the size of the dictionary is usually reduced, replacing the rare words with a special "UNK" token. Therefore, if a new word occurs during testing, it can be assumed that it is rare (since it never appears in the training set) and replace it with "UNK".


Instruction: A proper way to adjust input size of CNN (e.g. VGG)

Input: I want to train VGG on 128x128-sized images. I don't want to rescale them to 224x224 to save GPU-memory and training time. What would be the proper way to do so?

Output:
The best way is to keep the convolutional part as it is and replace the fully connected layers. This way it is even possible to take pretrained weights for the convolutional part of the network. The fully connected layers must be randomly initialized. This way one can finetune a network with a smaller input size.

Here some pytorch code

import torch
from torch.autograd import Variable
import torchvision
import torch.nn as nn

from torchvision.models.vgg import model_urls

VGG_TYPES = {'vgg11' : torchvision.models.vgg11,
             'vgg11_bn' : torchvision.models.vgg11_bn,
             'vgg13' : torchvision.models.vgg13,
             'vgg13_bn' : torchvision.models.vgg13_bn,
             'vgg16' : torchvision.models.vgg16,
             'vgg16_bn' : torchvision.models.vgg16_bn,
             'vgg19_bn' : torchvision.models.vgg19_bn,
             'vgg19' : torchvision.models.vgg19}

class Custom_VGG(nn.Module):

    def __init__(self,
                 ipt_size=(128, 128),
                 pretrained=True,
                 vgg_type='vgg19_bn',
                 num_classes=1000):
        super(Custom_VGG, self).__init__()

        # load convolutional part of vgg
        assert vgg_type in VGG_TYPES, "Unknown vgg_type '{}'".format(vgg_type)
        vgg_loader = VGG_TYPES[vgg_type]
        vgg = vgg_loader(pretrained=pretrained)
        self.features = vgg.features

        # init fully connected part of vgg
        test_ipt = Variable(torch.zeros(1,3,ipt_size[0],ipt_size[1]))
        test_out = vgg.features(test_ipt)
        self.n_features = test_out.size(1) * test_out.size(2) * test_out.size(3)
        self.classifier = nn.Sequential(nn.Linear(self.n_features, 4096),
                                        nn.ReLU(True),
                                        nn.Dropout(),
                                        nn.Linear(4096, 4096),
                                        nn.ReLU(True),
                                        nn.Dropout(),
                                        nn.Linear(4096, num_classes)
                                       )
        self._init_classifier_weights()

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

    def _init_classifier_weights(self):
        for m in self.classifier:
            if isinstance(m, nn.Linear):
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()

To create a vgg just call this:

vgg = Custom_VGG(ipt_size=(128, 128), pretrained=True)