Instruction: Need help understand this implementation of ConvLSTM code in pytorch

Input:
I am having issue understand the following implementation of ConvLSTM. I don't really understand what input_size + hidden_size is? Also the 4 * hidden_size value for output? model = ConvLSTMCell(c, d) tells us that c and d are input_size and hidden_size which are 3 and 5 respectively. I assume c is channel and d stands for output dimension? Also, why are we multiplying hidden_size by 4 for output? Is 4 the default 4 gates of the LSTM cell? Can some one more experience explain to me what is going on within the convolution? Thank you.

self.Gates = nn.Conv2d(input_size + hidden_size, 4 * hidden_size,
KERNEL_SIZE, padding=PADDING)

import torch
from torch import nn
import torch.nn.functional as f
from torch.autograd import Variable

# Define some constants
KERNEL_SIZE = 3
PADDING = KERNEL_SIZE // 2

class ConvLSTMCell(nn.Module):
    """
    Generate a convolutional LSTM cell
    """

    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.Gates = nn.Conv2d(input_size + hidden_size, 4 * hidden_size, KERNEL_SIZE, padding=PADDING)

    def forward(self, input_, prev_state):

        # get batch and spatial sizes
        batch_size = input_.data.size()[0]
        spatial_size = input_.data.size()[2:]

        # generate empty prev_state, if None is provided
        if prev_state is None:
            state_size = [batch_size, self.hidden_size] + list(spatial_size)
            prev_state = (
                Variable(torch.zeros(state_size)),
                Variable(torch.zeros(state_size))
            )

        prev_hidden, prev_cell = prev_state

        # data size is [batch, channel, height, width]
        stacked_inputs = torch.cat((input_, prev_hidden), 1)
        gates = self.Gates(stacked_inputs)

        # chunk across channel dimension
        in_gate, remember_gate, out_gate, cell_gate = gates.chunk(4, 1)

        # apply sigmoid non linearity
        in_gate = f.sigmoid(in_gate)
        remember_gate = f.sigmoid(remember_gate)
        out_gate = f.sigmoid(out_gate)

        # apply tanh non linearity
        cell_gate = f.tanh(cell_gate)

        # compute current cell and hidden state
        cell = (remember_gate * prev_cell) + (in_gate * cell_gate)
        hidden = out_gate * f.tanh(cell)

        return hidden, cell

def _main():
    """
    Run some basic tests on the API
    """

    # define batch_size, channels, height, width
    b, c, h, w = 1, 3, 4, 8
    d = 5           # hidden state size
    lr = 1e-1       # learning rate
    T = 6           # sequence length
    max_epoch = 20  # number of epochs

    # set manual seed
    torch.manual_seed(0)

    print('Instantiate model')
    model = ConvLSTMCell(c, d)
    print(repr(model))

    print('Create input and target Variables')
    x = Variable(torch.rand(T, b, c, h, w))
    y = Variable(torch.randn(T, b, d, h, w))

    print('Create a MSE criterion')
    loss_fn = nn.MSELoss()

    print('Run for', max_epoch, 'iterations')
    for epoch in range(0, max_epoch):
        state = None
        loss = 0
        for t in range(0, T):
            state = model(x[t], state)
            loss += loss_fn(state[0], y[t])

        print(' > Epoch {:2d} loss: {:.3f}'.format((epoch+1), loss.data[0]))

        # zero grad parameters
        model.zero_grad()

        # compute new grad parameters through time!
        loss.backward()

        # learning_rate step against the gradient
        for p in model.parameters():
            p.data.sub_(p.grad.data * lr)

    print('Input size:', list(x.data.size()))
    print('Target size:', list(y.data.size()))
    print('Last hidden state size:', list(state[0].size()))

if __name__ == '__main__':
    _main()

__author__ = "Alfredo Canziani"
__credits__ = ["Alfredo Canziani"]
__maintainer__ = "Alfredo Canziani"
__email__ = "alfredo.canziani@gmail.com"
__status__ = "Prototype"  # "Prototype", "Development", or "Production"
__date__ = "Jan 17"

Output: Yes, you are right, the output is timed by 4 because there are 4 gates


Instruction: How to convert numpy array into LongTensor type Variable in PyTorch?

Input:
I am trying to convert numpy array into PyTorch LongTensor type Variable as follows:

import numpy as np
import torch as th

y = np.array([1., 1., 1.1478225, 1.1478225, 0.8521775, 0.8521775, 0.4434675])
yth = Variable(th.from_numpy(y)).type(torch.LongTensor)

However the result I am getting is a rounded off version:

tensor([ 1,  1,  1,  1,  0,  0,  0])

How can I keep the precision of numpy array while getting LongTensor variable?

Expected result should be:

tensor([1., 1., 1.1478225, 1.1478225, 0.8521775, 0.8521775, 0.4434675])

Output:
LongTensor represents tensors with values of type long / int64 (c.f. table in doc). Your float values are thus converted (i.e. rounded) to integers.

To keep float values, use FloatTensor (float32) or DoubleTensor (float64) instead.


Instruction: Cannot load torchvision despite it being installed

Input:
I have installed pytorch and torchvision using:

conda install pytorch-cpu -c pytorch
pip install torchvision

when I try to run the following in spyder:

import torch
import torchvision
import torchvision.transforms as transforms

I get:

Traceback (most recent call last):

  File "<ipython-input-2-0bf25e9dac67>", line 2, in <module>
    import torchvision

  File "C:\Users\lkoefoed\AppData\Local\Continuum\anaconda3\lib\site-package\torchvision\__init__.py", line 2, in <module>
from torchvision import datasets

  File "C:\Users\lkoefoed\AppData\Local\Continuum\anaconda3\lib\site-packages\torchvision\datasets\__init__.py", line 1, in <module>
from .lsun import LSUN, LSUNClass

  File "C:\Users\lkoefoed\AppData\Local\Continuum\anaconda3\lib\site-packages\torchvision\datasets\lsun.py", line 2, in <module>
from PIL import Image

  File "C:\Users\lkoefoed\AppData\Local\Continuum\anaconda3\lib\site-packages\PIL\Image.py", line 56, in <module>
from . import _imaging as core

ImportError: DLL load failed: The specified module could not be found.

Output:
Fixed by running:

conda install pytorch-cpu -c pytorch
pip install torchvision

Deleting the PIL and pillow folders in site-packages, then running:

pip install pillow


Instruction: PyToch Skorch passing 3 dimensional input

Input:
Im trying to fit a model in PyTorch while using skorch. My problem is, that my model uses an LSTM layer, which expects 3d input and I don't know how to pass the input correctly.

When passing a 2d-array to fit() I get an error from PyTorch for the expected 3d input. If passing a 3d-array, I get the error from the fit() method for inconsistent lengths (which both make perfectly sense to me).

Example code below:

import numpy as np
import torch
from torch import nn
import skorch

class lstmNet(nn.Module):

    def __init__(self, input_size, hidden_size, num_layers):
        super(lstmNet, self).__init__()
        self.rnn = nn.LSTM(
            input_size,
            hidden_size,
            num_layers        )
        self.lin = nn.Linear(
            in_features=hidden_size,
            out_features=1
        )

    def forward(self, x):
        print(x.size())
        out, hn = self.rnn(x)
        out = self.lin(out)
        return out

input_feat = 5
hidden_size = 10
lstmLayers = 2

seq = 20
batch = 30
features = 5

net = skorch.NeuralNet(
    module=lstmNet(
        input_size=input_feat,
        hidden_size=hidden_size,
        num_layers=lstmLayers
    ),
    criterion=torch.nn.MSELoss,
    optimizer=torch.optim.SGD,
    lr=0.1,
    max_epochs=10
)

#inputArr2d = np.random.rand(seq * batch, features)
inputArr3d = np.random.rand(seq, batch, features)
print('input:\n {}\nshape: {}'.format(inputArr3d, inputArr3d.shape))
targetArr = np.random.rand((seq * batch))
#print('target:\n {}\nshape: {}'.format(targetArr, targetArr.shape))

net.fit(X=inputArr3d, y=targetArr)

This is the error when calling net.fit(X=inputArr2d, y=targetArr):

Traceback (most recent call last):
File "C:\Spielplatz\Python\examples\playground.py", line 64, in <module>
net.fit(X=inputArr2d, y=targetArr)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 686, in fit
self.partial_fit(X, y, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 646, in partial_fit
self.fit_loop(X, y, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 584, in fit_loop
step = self.train_step(Xi, yi, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 507, in train_step
y_pred = self.infer(Xi, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 810, in infer
return self.module_(x, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 491, in __call__
result = self.forward(*input, **kwargs)
File "C:\Spielplatz\Python\examples\playground.py", line 33, in forward
out, hn = self.rnn(x)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 491, in __call__
result = self.forward(*input, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\rnn.py", line 178, in forward
self.check_forward_args(input, hx, batch_sizes)
File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\rnn.py", line 126, in check_forward_args
expected_input_dim, input.dim()))
RuntimeError: input must have 3 dimensions, got 2

And this is the error when calling net.fit(X=inputArr3d, y=targetArr):

Traceback (most recent call last):
File "C:\Spielplatz\Python\examples\playground.py", line 64, in <module>
net.fit(X=inputArr3d, y=targetArr)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 686, in fit
self.partial_fit(X, y, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 646, in partial_fit
self.fit_loop(X, y, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 573, in fit_loop
X, y, **fit_params)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 1004, in get_split_datasets
dataset = self.get_dataset(X, y)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\net.py", line 961, in get_dataset
return dataset(X, y, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\skorch\dataset.py", line 104, in __init__
raise ValueError("X and y have inconsistent lengths.")
ValueError: X and y have inconsistent lengths.

Output:
Passing a 2D-array is, as you said yourself, not going to work since you don't have a time dimension then, so the error message is correct in this case (LSTM expects 3 dimensions).

The confiusing part is the error message when passing a 3D array. What is happening is that you are formatting your data as (sequence, batch, feature), i.e. "batch second". There is an alternative format called "batch first" where the data is formatted as (batch, sequence, feature) which is the default in sklearn (and consequently in skorch). If you re-format your data this way and configure your LSTM to use this format (pass batch_first=True) the error should go away:

    self.rnn = nn.LSTM(
        input_size,
        hidden_size,
        num_layers,
        batch_first=True,
    )
# ...
inputArr3d = np.random.rand(batch, seq, features)

Note that your module also needs to convert the return value of the RNN
into something that the following layers can process (i.e. flatten
the time dimension) or otherwise the linear layer will not know what
to do with the extra time dimension.


Instruction: Optimizer got an empty parameter list in pytorch

Input:
I try to use convolutional network structures in a VAE experiment, the code is something like:

Class CNN(nn.Module):...

hparams1=[...]
hparams2=[...]

Model1=CNN(hyparams1)
Model2=CNN(hyparams2)

def VAE(x):
   encoder=Model1(x)
   z=reparameterize(encoder)  #reparameterize is defined
   decoder=Model2(z)
   return decoder

if args.cuda:
    Model1.cuda()
    Model2.cuda()

parameters = chain(Model1.parameters(), Model2.parameters())
optimizer = optim.Adam(parameters, lr=1e-3)

Error:Optimizer got an empty parameter list.

The same problem occurs when I defind VAE as a class:

class VAE(nn.Module):
  def encoder(x,model):
     return model(x)

  def reparameterize():

  def decoder(z,model):
     return model(z)

  def forward(x,Model1,Model2)
     output=encoder(x,Model1)
     z=reparameterize(output)
     return decoder(z,Model2)

model=VAE()
optimizer = optim.Adam(model.parameters(), lr=1e-3)

Error:Optimizer got an empty parameter list.

I guess maybe the problem is that I fail to initialize the parameters in a class VAE, but I don't know how to deal with it. Thank you very much!

Output:
In the first case, you can use:

parameters = list(Model1.parameters())+ list(Model2.parameters())
optimizer = optim.Adam(parameters, lr=1e-3)

In the second case, you didn't create the object, so basically you can try this:

model = VAE()
optimizer = optim.Adam(model.parameters(), lr=1e-3)

By the way, you can start from modifying the VAE example provided by Pytorch.

EDIT

Perhaps you miss the initial function or initialize the model in a wrong way. See the init function here.

Hope it helps.


Instruction: Pytorch, Unable to get repr for

Input:
I'm implementing some RL in PyTorch and had to write my own mse_loss function (which I found on Stackoverflow ;) ).
The loss function is:

def mse_loss(input_, target_):
    return torch.sum(
        (input_ - target_) * (input_ - target_)) / input_.data.nelement()

Now, in my training loop, the first input is something like:

tensor([-1.7610e+10]), tensor([-6.5097e+10])

With this input I'll get the error:

Unable to get repr for <class 'torch.Tensor'>

Computing a = (input_ - target_) works fine, while b = a * a respectively b = torch.pow(a, 2) will fail with the error metioned above.

Does anyone know a fix for this?

Thanks a lot!

Update:
I just tried using torch.nn.functional.mse_loss which will result in the same error..

Output:
I had the same error,when I use the below code

criterion = torch.nn.CrossEntropyLoss().cuda()
output=output.cuda()
target=target.cuda()
loss=criterion(output, target)

but I finally found my wrong：output is like tensor([[0.5746,0.4254]]) and target is like tensor([2]),the number 2 is out of indice of output

when I not use GPU,this error message is:

RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed.  at /opt/conda/conda-bld/pytorch-nightly_1547458468907/work/aten/src/THNN/generic/ClassNLLCriterion.c:93


Instruction: Pytorch loss inf nan

Input:
I'm trying to do simple linear regression with 1 feature. It's a simple 'predict salary given years experience' problem.
The NN trains on years experience (X) and a salary (Y).
For some reason the loss is exploding and ultimately returns inf or nan
This is the code I have:
    import torch
    import torch.nn as nn
    import pandas as pd
    import numpy as np

    dataset = pd.read_csv('./salaries.csv')

    x_temp = dataset.iloc[:, :-1].values
    y_temp = dataset.iloc[:, 1:].values

    X_train = torch.FloatTensor(x_temp)
    Y_train = torch.FloatTensor(y_temp)

    class Model(torch.nn.Module):
        def __init__(self):
            super().__init__()
            self.linear = torch.nn.Linear(1,1)

        def forward(self, x):
            y_pred = self.linear(x)
            return y_pred

    model = Model()

    loss_func = torch.nn.MSELoss(size_average=False)
    optim = torch.optim.SGD(model.parameters(), lr=0.01)

    #training
    for epoch in range(200):
        #calculate y_pred
        y_pred = model(X_train)

        #calculate loss
        loss = loss_func(y_pred, Y_train)
        print(epoch, "{:.2f}".format(loss.data))

        #backward pass + update weights
        optim.zero_grad()
        loss.backward()
        optim.step()

    test_exp = torch.FloatTensor([[8.0]])
    print("8 years experience --> ", model(test_exp).data[0][0].item())

As I mentioned, once it starts training the loss gets super big and ends up showing inf after like the 10th epoch.
I suspect it may have something to do with how i'm loading the data? This is what is in salaries.csv file:
Years Salary
1.1 39343
1.3 46205
1.5 37731
2   43525
2.2 39891
2.9 56642
3   60150
3.2 54445
3.2 64445
3.7 57189
3.9 63218
4   55794
4   56957
4.1 57081
4.5 61111
4.9 67938
5.1 66029
5.3 83088

Thank you for your help

Output:
Once the loss becomes inf after a certain pass, your model gets corrupted after backpropagating. This probably happens because the values in "Salary" column are too big. try normalizing the salaries.

Alternatively, you could try to initialize the parameters by hand (rather than letting it be initialized randomly), letting the bias term be the average of salaries, and the slope of the line be 0 (for instance). That way the initial model would be close enough to the optimal solution, so that the loss does not blow up.


Instruction: Pytorch inceptionV3 transfer learning gives error - max() received an invalid combination of arguments

Input:
The program for transfer learning inception_v3 in pytorch that i am using is here : https://drive.google.com/file/d/1zn4z7nOp_wJne0En6zq4WJfwHVVftERT/view?usp=sharing

I am getting the following error upon running the program :

Epoch 0/24
    ---------------------------------------------------------------------------
    TypeError                                 Traceback (most recent call last)
    <ipython-input-20-cc88ea5f8bd3> in <module>()
          1 model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
    ----> 2                        num_epochs=25)

    <ipython-input-17-812cf3c4576a> in train_model(model, criterion, optimizer, scheduler, num_epochs)
         33                     outputs = model(inputs)
         34                     print(outputs)
    ---> 35                     _, preds = torch.max(outputs, 1)
         36                     loss = criterion(outputs, labels)
         37

    TypeError: max() received an invalid combination of arguments - got (tuple, int), but expected one of:
     * (Tensor input)
     * (Tensor input, Tensor other, Tensor out)
     * (Tensor input, int dim, bool keepdim, tuple of Tensors out)

How can this be fixed ?
Thank you

Output:
in this case, I have changed the code as below and its worked for me,
Tutorial https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html

model_ft = models.inception_v3(pretrained=True)
model_ft.aux_logits=False


Instruction: Different behavior with torch.mean(data, 0) and torch.mean(data) for vector

Input:
I was surprised to find different behavior with torch.mean(data, 0) and torch.mean(data), where "data" — is 1D-tensor (vector, not a matrix or something else):

from torchvision import datasets
import torch

path = './MNIST_data'

data = datasets.MNIST(path, train=True, download=True).train_data.view(-1).float()

print(torch.mean(data))
print(torch.mean(data, 0))

Results after execution:

tensor(33.3184)
tensor(33.4961)

Could anyone suppose what is going on? I assumed that the results should be the same.

Output:
An Example should help you clear your doubt.

Lets say we have data = torch.Tensor([[1,2,3,4],[1,2,3,4]])

Now When you perform torch.mean(data),It will sum all the elements in the data tensor and divide that by the number of elements in that tensor,Giving you a result of 2.5

As for your operation of torch.mean(data, 0) This will perform mean along the horizontal direction,which means it will take the first element of row one which is 1 and take the first element of row 2 which is 2,sum them up and divide by 2.It helps to visualize the data array this way

[1, 2, 3, 4]
[1, 2, 3, 4]

The final result will be tensor of [1, 2, 3, 4] if you know how we ended up getting this tensor, You've understood the difference.

Hope that clears it, Let me know if you have questions


Instruction: RuntimeError: Expected object of type torch.FloatTensor but found type torch.cuda.DoubleTensor for argument #2 'weight'

Input:
I have been trying to retrain a model but unfortunately the last 2 days I keep getting the same error.

Could you please help a little bit with this one?

Initial work:

%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import matplotlib.pyplot as plt
import numpy as np
import time

import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torch.autograd import Variable
from torchvision import datasets, transforms
import torchvision.models as models
from collections import OrderedDict

Datasets:

data_dir = 'flowers'
train_dir = data_dir + '/train'

data_dir = 'flowers'

train_transforms = transforms.Compose([transforms.Resize(224),
                                       transforms.RandomResizedCrop(224),
                                       transforms.RandomRotation(45),
                                       transforms.RandomHorizontalFlip(),
                                       transforms.ToTensor(),
                                       transforms.Normalize([0.485, 0.456, 0.406],
                                                            [0.229, 0.224, 0.225])])

train_data = datasets.ImageFolder(train_dir, transform=train_transforms)

trainloader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True)

import json

with open('cat_to_name.json', 'r') as f:
    cat_to_name = json.load(f)

Tried to use a pretrained model and train only the classifier:

# Load a pretrained model
model = models.vgg16(pretrained=True)

# Keep the parameters the same
for param in model.parameters():
    param.requires_grad = False

# and final output 102, since tht we have 102 flowers.
classifier = nn.Sequential(OrderedDict([
                          ('fc1', nn.Linear(25088, 4096)),
                          ('relu', nn.ReLU()),
                          ('fc3', nn.Linear(4096, 102)),
                          ('output', nn.LogSoftmax(dim=1))
                          ]))

# Replace model's old classifier with the new classifier
model.classifier = classifier

# Calculate the loss
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)

model.to('cuda')

epochs = 1
print_every = 40
steps = 0

for e in range(epochs):
    running_loss = 0
    model.train()
   # model = model.double()
    for images, labels in iter(trainloader):
        steps += 1

        images.resize_(32, 3, 224, 224)

        inputs = Variable(images.to('cuda'))
        targets = Variable(labels.to('cuda'))

        optimizer.zero_grad()

        # Forward and backward passes
        output = model.forward(images)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()

        #running_loss += loss.data[0]
        running_loss += loss.item()

        if steps % print_every == 0:
            print("Epoch: {}/{}... ".format(e+1, epochs),
                  "Loss: {:.4f}".format(running_loss/print_every))

Error message:

  RuntimeError: Expected object of type torch.FloatTensor but found type torch.cuda.DoubleTensor for argument #2 weight

Output:
  you should have passed inputs to the feed forward network but you have passed images to the network

# Forward and backward passes
output = model.forward(inputs)


Instruction: pytorch: zero_grad vs zero_grad() - role of parenthesis?

Input:
I'm trying to train a pytorch model as follows:

start = time.time()

for epoch in range(100):

    t_loss = 0
    for i in range(100):

        optimizer.zero_grad
        scores = my_model(sent_dict_list[i])
        scores = scores.permute(0, 2, 1)

        loss = loss_function(scores, torch.tensor(targ_list[i]).cuda())
        t_loss += loss.item()
        loss.backward()
        optimizer.step()

    print("t_loss = ", t_loss)

I find that when I call "optimizer.zero_grad" my loss decreases at the end of every epoch whereas when I call "optimizer.zero_grad()" with the parenthesis it stays almost exactly the same. I don't know what difference this makes and was hoping someone could explain it to me.

Output:
I assume you're new to python, the '()' means simple a function call.
Consider this example:

>>> def foo():
       print("function")

>>> foo
<function __main__.foo>
>>> foo()
function

Remember functions are objects in python, you can even store them like this:

>>> [foo, foo, foo]

Returning to your question, you have to call the function otherwise it won't work.


Instruction: How to compute Pairwise L1 Distance matrix on very large images in neighborhood only?

Input:
I am working on Deep learning approach for my project. And I need to calculate Distance Matrix on 4D Tensor which will be of size N x 128 x 64 x 64 (Batch Size x Channels x Height x Width). The distance matrix for this type of tensor will of size N x 128 x 4096 x 4096 and it will be impossible to fit this type of tensor in GPU, even on CPU it will require lot of memory. So, I would like to calculate the distance matrix only in some neighborhood pixels (say within radius of 5) and consider this rectangular matrix for further processing in neural network. With this approach my distance matrix will be of size N x 128 x 4096 x 61. It will take less memory in comparison to full distance matrix.

Precisely, I am trying to implement the Convolution Random Walk Networks for Semantic Segmentation. This network needs to calculate the Pairwise L1 Distance for features.

Architecture

Just to add this type of Distance Matrix is usually calculated for image segmentation via spectral clustering.

For Example

X = [[a,b],[c,d]]

L1_dist = [ [0, |a-b|, |a-c|, 0],
            [|a-b|, 0, 0, |b-d|],
            [|a-c|, 0, 0, |c-d| ],
            [0,    |b-d|, |c-d|, 0 ]
          ]

Final_L1_dist = [ [0, |a-b|, |a-c|], // "a" is near to b and c. Including self element i.e. a
            [|a-b|, 0, |b-d|], // "b" is near to a and d.
            [|a-c|, 0, |c-d| ], // "c" is near to a and d.
            [|b-d|, |c-d|, 0 ] // "d" is near to b and c.
          ]

I would appreciate, if some one can help me to find an efficient way to compute such a matrix.

Thanks

Output:
As far as I understand, the goal is to apply minus operation to each pixel and its surrounding neighbors. This sounds like convolution to me.

Consider the following convolution process (assume padding='SAME'):

The 3x3 kernel calculates, for each pixel, the difference between the center pixel and its left one. For other neighbors, consider the following kernels:

Thus the goal can be achieved via the following:

Repeat each kernel for num_channels times using tf.tile;
Apply each kernel channel-wisely using tf.nn.depthwise_conv2d;
Do tf.abs to get the distance;
Reshape each distance tensor to NxCx(HW)x1 and stack them properly.

For efficient for loop, you may consider using tf.map_fn.


Instruction: How to train Actor-Critic (A2C) reinforcement learning

Input:
I am currently been able to train a system using Q-Learning. I will to move it to Actor_Critic (A2C) method. Please don't ask me why for this move, I have to.

I am currently borrowing the implementation from https://github.com/higgsfield/RL-Adventure-2/blob/master/1.actor-critic.ipynb

The thing is, I am keep getting a success rate of approx ~ 50% (which is basically random behavior). My game is a long episode (50 steps). Should I print out the reward, the value, or what? How should I debug this?

Here are some log:

simulation episode 2: Success, turn_count =20
loss = tensor(1763.7875)

simulation episode 3: Fail,  turn_count= 42
loss = tensor(44.6923)

simulation episode 4: Fail,  turn_count= 42
loss = tensor(173.5872)

simulation episode 5: Fail,  turn_count= 42
loss = tensor(4034.0889)

simulation episode 6: Fail,  turn_count= 42
loss = tensor(132.7567)

loss = simulation episode 7: Success, turn_count =22
loss = tensor(2099.5344)

As a general trend, I have observed that for Success episodes, the loss tends to be huge, where as for Fail episode, the loss function output tends to be small.

Output:
I think you're making a mistake, if you really want to know how to implement Actor Critic algorithm you need first to master 2 things :
- Implementing value based RL algorithms (such as DQN).
- Implementing policy based RL algorithms (such as Policy Gradients).

You can't directly jump on actor critic models, in fact you can but you will understand nothing if you're not able to understand actor (policy based) and critic (value based) separately.

It's like if you wanted to paint the Joconde before beginning by learning how to paint.

My advice, take the time to learn these 2 elements before implementing an AC agent.

I made a free course with tensorflow and completes implementations here https://simoninithomas.github.io/Deep_reinforcement_learning_Course/

But again, implement architectures by yourself, copy an architecture is useless if you don't really understand it.


Instruction: Add / substract between matrix and vector in pytorch

Input:
I want to do + / - / *  between matrix and vector in pytorch.
How can I do with good performance?
I tried to use expand, but it's really slow (I am using big matrix with small vector).

a = torch.rand(2,3)
print(a)
 0.7420  0.2990  0.3896
 0.0715  0.6719  0.0602
[torch.FloatTensor of size 2x3]
b = torch.rand(2)
print(b)
 0.3773
 0.6757
[torch.FloatTensor of size 2]
a.add(b)
Traceback (most recent call last):
  File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3066, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-17-a1cb1b03d031>", line 1, in <module>
    a.add(b)
RuntimeError: inconsistent tensor size, expected r_ [2 x 3], t [2 x 3] and src [2] to have the same number of elements, but got 6, 6 and 2 elements respectively at c:\miniconda2\conda-bld\pytorch-cpu_1519449358620\work\torch\lib\th\generic/THTensorMath.c:1021

Expected result:

 0.7420-0.3773  0.2990-0.3773  0.3896-0.3773
 0.0715-0.6757  0.6719-0.6757  0.0602-0.6757

Output:
To make use of broadcasting, you need to promote the dimension of the tensor b to two dimensions since the tensor a is 2D.

In [43]: a
Out[43]:
tensor([[ 0.9455,  0.2088,  0.1070],
        [ 0.0823,  0.6509,  0.1171]])

In [44]: b
Out[44]: tensor([ 0.4321,  0.8250])

# subtraction
In [46]: a - b[:, None]
Out[46]:
tensor([[ 0.5134, -0.2234, -0.3252],
        [-0.7427, -0.1741, -0.7079]])

# alternative way to do subtraction
In [47]: a.sub(b[:, None])
Out[47]:
tensor([[ 0.5134, -0.2234, -0.3252],
        [-0.7427, -0.1741, -0.7079]])

# yet another approach
In [48]: torch.sub(a, b[:, None])
Out[48]:
tensor([[ 0.5134, -0.2234, -0.3252],
        [-0.7427, -0.1741, -0.7079]])

The other operations (+, *) can be done analogously.

In terms of performance, there seems to be no advantage of using one approach over the others. Just use any one of the three approaches.

In [49]: a = torch.rand(2000, 3000)
In [50]: b = torch.rand(2000)

In [51]: %timeit torch.sub(a, b[:, None])
2.4 ms ± 8.31 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [52]: %timeit a.sub(b[:, None])
2.4 ms ± 6.94 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [53]: %timeit a - b[:, None]
2.4 ms ± 12 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)


Instruction: LSTM Sequence length

Input:
I want to ask if there is an optimal sequence length of a LSTM network in general, or in terms of time series prediction problems?
I read about vanishing gradient or exploding gradient problems that very long RNN networks had and LSTM tried to solve and succeeded to a certain extent.
I also heard about techniques to handle very large sequences with LSTM’s and RNN’s in general like: truncating sequences, summarizing sequences, truncating backpropagation through time or even using an Encoder-Decoder architecture.
I asked this question because I didn’t find a research paper about this, only this blog post that stated an optimal sequence length between 10-30.

Output:
Do some Model Selection.
TLDR: Just try it out.
Because training is already very computationally expensive, the easiest way to calculate how successful a model would be is to test it out. The combination that works best cannot be easily predetermined, especially not with such a vague description (or no description at all) of how the actual problem looks like.
From this answer:

It totally depends on the nature of your data and the inner correlations, there is no rule of thumb. However, given that you have a large amount of data a 2-layer LSTM can model a large body of time series problems / benchmarks.

So in your case, you might want to try sequence lengths from 10 - 30. But I'd also try and evaluate how your training algorithm performs outside of that recommendation by the post you linked.


Instruction: Net does not change weights during training, pytorch

Input:
I'm implementing DDPG and got stuck training my two nets.

All in all I've got 4 nets called: actor, actor_target, critic and critic_target.
I'm training the actor and critic in the training loop and do soft updates for the other two nets with:

def update_weights(self, source, tau):
        for target, source in zip(self.parameters(), source.parameters()):
            target.data.copy_(tau * source.data + (1 - tau) * target.data)

My training loop looks like:

tensor_next_states = torch.tensor(next_states).view(-1, 1)
prediction_target = self.actor_target(tensor_next_states).data.numpy()
target_critic_output = self.critic_target(
    construct_tensor(next_states, prediction_target))
y = torch.tensor(rewards).view(-1,1) + \
    self.gamma * target_critic_output
output_critic = self.critic(
    torch.tensor(construct_tensor(states, actions), dtype=torch.float))

# compute loss and update critic
self.critic.zero_grad()
loss_critic = self.criterion_critic(y, output_critic)
loss_critic.backward()
self.critic_optim.step()

# compute loss and update actor
tensor_states = torch.tensor(states).view(-1, 1)
ouput_actor = self.actor(tensor_states).data.numpy()
self.actor.zero_grad()
loss_actor = (-1.) * \
             self.critic(construct_tensor(states, ouput_actor)).mean()
loss_actor.backward()
self.actor_optim.step()

# update target
self.actor_target.update_weights(self.actor, self.tau)
self.critic_target.update_weights(self.critic, self.tau)

using SGD as optimizer and self.criterion_critic = F.mse_loss.

construct_tensor(a,b) constructs a tensor like [a[0], b[0], a[1], b[1], ...].

I've noticed, that the RMSE on the test set before and after training is the same.
So I debugged a lot and noticed in update_weights that the weights of the trained net and the target net are the same - so I concluded, that the training does not have any affect on the weights of the trained net.
I already checked that the computed losses are not zero but still a float, checked replacing the zero_grad() calls and moving the computed losses to self, which hadn't got any impact.

Does anyone already met this behavior and/or has any tips or know how to fix this?

Update: Full code:

import datetime
import random
from collections import namedtuple

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

def combine_tensors(s, a):
    """
    Combines the two given tensors
    :param s: tensor1
    :param a: tensor2
    :return: combined tensor
    """
    target = []
    if not len(a[0].shape) == 0:
        for i in range(len(s)):
            target.append(torch.cat((s[i], a[i])).data.numpy())
    else:
        for i in range(len(s)):
            target.append(torch.cat((s[i], a[i].float().view(-1))) \
                          .data.numpy())
    return torch.tensor(target, device=device)

class actor(nn.Module):
    """
    Actor - gets a state (2-dim) and returns probabilities about which
    action to take (4 actions -> 4 outputs)
    """

    def __init__(self):
        super(actor, self).__init__()

        # define net structure
        self.input_layer = nn.Linear(2, 4)
        self.hidden_layer_1 = nn.Linear(4, 8)
        self.hidden_layer_2 = nn.Linear(8, 16)
        self.hidden_layer_3 = nn.Linear(16, 32)
        self.output_layer = nn.Linear(32, 4)

        # initialize them
        nn.init.xavier_uniform_(self.input_layer.weight)
        nn.init.xavier_uniform_(self.hidden_layer_1.weight)
        nn.init.xavier_uniform_(self.hidden_layer_2.weight)
        nn.init.xavier_uniform_(self.hidden_layer_3.weight)
        nn.init.xavier_uniform_(self.output_layer.weight)

        nn.init.constant_(self.input_layer.bias, 0.1)
        nn.init.constant_(self.hidden_layer_1.bias, 0.1)
        nn.init.constant_(self.hidden_layer_2.bias, 0.1)
        nn.init.constant_(self.hidden_layer_3.bias, 0.1)
        nn.init.constant_(self.output_layer.bias, 0.1)

    def forward(self, state):
        state = F.relu(self.input_layer(state))
        state = F.relu(self.hidden_layer_1(state))
        state = F.relu(self.hidden_layer_2(state))
        state = F.relu(self.hidden_layer_3(state))
        state = F.softmax(self.output_layer(state), dim=0)
        return state

class critic(nn.Module):
    """
    Critic - gets a state (2-dim) and an action and returns value
    """

    def __init__(self):
        super(critic, self).__init__()
        # define net structure
        self.input_layer = nn.Linear(3, 8)
        self.hidden_layer_1 = nn.Linear(8, 16)
        self.hidden_layer_2 = nn.Linear(16, 32)
        self.hidden_layer_3 = nn.Linear(32, 16)
        self.output_layer = nn.Linear(16, 1)

        # initialize them
        nn.init.xavier_uniform_(self.input_layer.weight)
        nn.init.xavier_uniform_(self.hidden_layer_1.weight)
        nn.init.xavier_uniform_(self.hidden_layer_2.weight)
        nn.init.xavier_uniform_(self.hidden_layer_3.weight)
        nn.init.xavier_uniform_(self.output_layer.weight)

        nn.init.constant_(self.input_layer.bias, 0.1)
        nn.init.constant_(self.hidden_layer_1.bias, 0.1)
        nn.init.constant_(self.hidden_layer_2.bias, 0.1)
        nn.init.constant_(self.hidden_layer_3.bias, 0.1)
        nn.init.constant_(self.output_layer.bias, 0.1)

    def forward(self, state_, action_):
        state_ = combine_tensors(state_, action_)
        state_ = F.relu(self.input_layer(state_))
        state_ = F.relu(self.hidden_layer_1(state_))
        state_ = F.relu(self.hidden_layer_2(state_))
        state_ = F.relu(self.hidden_layer_3(state_))
        state_ = self.output_layer(state_)
        return state_

Transition = namedtuple('Transition',
                        ('state', 'action', 'next_state', 'reward'))

class ReplayMemory(object):
    """
    Memory
    """

    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, *args):
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

def compute_action(actor_trainined, state, eps=0.1):
    """
    Computes an action given the actual policy, the state and an eps.
    Eps is resposible for the amount of exploring
    :param actor_trainined: actual policy
    :param state:
    :param eps: float in [0,1]
    :return:
    """
    denoise = random.random()

    if denoise > eps:
        action_probs = actor_trainined(state.float())
        return torch.argmax(action_probs).view(1).int()
    else:
        return torch.randint(0, 4, (1,)).view(1).int()

def compute_next_state(_action, _state):
    """
    Computes the next state given an action and a state
    :param _action:
    :param _state:
    :return:
    """
    state_ = _state.clone()
    if _action.item() == 0:
        state_[1] += 1
    elif _action.item() == 1:
        state_[1] -= 1
    elif _action.item() == 2:
        state_[0] -= 1
    elif _action.item() == 3:
        state_[0] += 1

    return state_

def update_weights(target, source, tau):
    """
    Soft-Update of weights
    :param target:
    :param source:
    :param tau:
    :return:
    """
    for target, source in zip(target.parameters(), source.parameters()):
        target.data.copy_(tau * source.data + (1 - tau) * target.data)

def update(transition__, replay_memory, batch_size_, gamma_):
    """
    Performs one update step
    :param transition__:
    :param replay_memory:
    :param batch_size_:
    :param gamma_:
    :return:
    """
    replay_memory.push(*transition__)

    if replay_memory.__len__() < batch_size_:
        return

    transitions = replay_memory_.sample(batch_size_)
    batch = Transition(*zip(*transitions))

    states = torch.stack(batch.state)
    actions = torch.stack(batch.action)
    rewards = torch.stack(batch.reward)
    next_states = torch.stack(batch.next_state)

    action_target = torch.argmax(actor_target(next_states.float()), 1).int()
    y = (
            rewards.float().view(-1, 1) +
            gamma_ * critic_target(next_states.float(), action_target.float())
            .float()
    )

    critic_trained.zero_grad()
    crit_ = critic_trained(states.float(), actions.float())
    # nn stuff does not work here! -> doing mse myself..
    # loss_critic = (torch.sum((y.float() - crit_.float()) ** 2.)
    #                / y.data.nelement())
    loss_critic = F.l1_loss(y.float(), crit_.float())
    loss_critic.backward()
    optimizer_critic.step()

    actor_trained.zero_grad()
    loss_actor = ((-1.) * critic_trained(states.float(),
                                         torch.argmax(
                                             actor_trained(states.float()), 1
                                         ).int().float())).mean()
    loss_actor.backward()
    optimizer_actor.step()

def get_eps(epoch):
    """
    Computes the eps for action choosing dependant on the epoch
    :param epoch: number of epoch
    :return:
    """
    if epoch <= 10:
        eps_ = 1.
    elif epoch <= 20:
        eps_ = 0.8
    elif epoch <= 40:
        eps_ = 0.6
    elif epoch <= 60:
        eps_ = 0.4
    elif epoch <= 80:
        eps_ = 0.2
    else:
        eps_ = 0.1
    return eps_

def compute_reward_2(state_, next_state_, terminal_state_):
    """
    Better (?) reward function that "compute_reward"
    If next_state == terminal_state -> reward = 100
    If next_state illegal           -> reward = -100
    if next_state is further away from terminal_state than state_ -> -2
    else 1
    :param state_:
    :param next_state_:
    :param terminal_state_:
    :return:
    """
    if torch.eq(next_state_, terminal_state_).all():
        reward_ = 100
    elif torch.eq(next_state_.abs(), 15).any():
        reward_ = -100
    else:
        if (state_.abs() > next_state_.abs()).any():
            reward_ = 1.
        else:
            reward_ = -2
    return torch.tensor(reward_, device=device, dtype=torch.float)

def compute_reward(next_state_, terminal_state_):
    """
    Computes some reward
    :param next_state_:
    :param terminal_state_:
    :return:
    """
    if torch.eq(next_state_, terminal_state_).all():
        return torch.tensor(100., device=device, dtype=torch.float)
    elif next_state_[0] == 15 or next_state_[1] == 15:
        return torch.tensor(-100., device=device, dtype=torch.float)
    else:
        return (-1.) * next_state_.abs().sum().float()

def fill_memory_2():
    """
    Fills the memory with random transitions which got a "good" action chosen
    """
    terminal_state_ = torch.tensor([0, 0], device=device, dtype=torch.int)
    while replay_memory_.__len__() < batch_size:
        state_ = torch.randint(-4, 4, (2,)).to(device).int()
        if state_[0].item() == 0 and state_[1].item == 0:
            continue

        # try to find a "good" action
        if state_[0].item() == 0:
            if state_[1].item() > 0:
                action_ = torch.tensor(1, device=device, dtype=torch.int)
            else:
                action_ = torch.tensor(0, device=device, dtype=torch.int)
        elif state_[1].item() == 0:
            if state_[0].item() > 0:
                action_ = torch.tensor(2, device=device, dtype=torch.int)
            else:
                action_ = torch.tensor(3, device=device, dtype=torch.int)
        else:
            random_bit = random.random()
            if random_bit > 0.5:
                if state_[1].item() > 0:
                    action_ = torch.tensor(1, device=device, dtype=torch.int)
                else:
                    action_ = torch.tensor(0, device=device, dtype=torch.int)
            else:
                if state_[0].item() > 0:
                    action_ = torch.tensor(2, device=device, dtype=torch.int)
                else:
                    action_ = torch.tensor(3, device=device, dtype=torch.int)

        action_ = action_.view(1).int()
        next_state_ = compute_next_state(action_, state_)
        reward_ = compute_reward_2(state_, next_state_, terminal_state_)

        transition__ = Transition(state=state_, action=action_,
                                  reward=reward_, next_state=next_state_)
        replay_memory_.push(*transition__)

def fill_memory():
    """
    Fills the memory with random transitions
    """
    while replay_memory_.__len__() < batch_size:
        state_ = torch.randint(-14, 15, (2,)).to(device).int()
        if state_[0].item() == 0 and state_[1].item == 0:
            continue
        terminal_state_ = torch.tensor([0, 0], device=device, dtype=torch.int)
        action_ = torch.randint(0, 4, (1,)).view(1).int()
        next_state_ = compute_next_state(action_, state_)
        reward_ = compute_reward_2(state_, next_state_, terminal_state_)

        transition__ = Transition(state=state_, action=action_,
                                  reward=reward_, next_state=next_state_)
        replay_memory_.push(*transition__)

if __name__ == '__main__':
    # get device if possible
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    # set seed
    seed_ = 0
    random.seed(seed_)  # seed of python
    if device == "cuda":
        # cuda seed
        torch.cuda.manual_seed(seed_)
    else:
        # cpu seed
        torch.manual_seed(seed_)

    # initialize the nets
    actor_trained = actor().to(device)
    actor_target = actor().to(device)
    # copy -> _trained eqaul _target
    actor_target.load_state_dict(actor_trained.state_dict())
    optimizer_actor = optim.RMSprop(actor_trained.parameters())
    # move them to the device
    critic_trained = critic().to(device)
    critic_target = critic().to(device)
    critic_target.load_state_dict((critic_trained.state_dict()))
    actor_target.load_state_dict((actor_trained.state_dict()))
    # used optimizer
    optimizer_critic = optim.RMSprop(critic_trained.parameters(),
                                     momentum=0.9, weight_decay=0.001)

    # replay memory
    capacity_replay_memory = 16384
    replay_memory_ = ReplayMemory(capacity_replay_memory)

    # hyperparams
    batch_size = 1024
    gamma = 0.7
    tau = 0.01
    num_epochs = 256

    # fill replay memory such that batching is possible
    fill_memory_2()

    # Print params
    printing_while_training = True
    printing_while_testing = False

    print('######################## Training ########################')
    starting_time = datetime.datetime.now()
    for i in range(num_epochs):
        # random state
        starting_state = torch.randint(-14, 15, (2,)).to(device).int()
        # skip if terminal state
        if starting_state[0].item() == 0 and starting_state[0].item() == 0:
            continue
        state = starting_state.clone()
        # terminal state
        terminal_state = torch.tensor([0, 0], device=device, dtype=torch.int)
        iteration = 0

        # get eps for exploring
        eps = get_eps(i)

        running_reward = 0.

        # training loos
        while True:
            # compute action and next state
            action = compute_action(actor_trained, state, eps)
            next_state = compute_next_state(action, state)

            # finished if next state is terminal state
            if torch.eq(next_state, terminal_state).all():
                reward = compute_reward_2(state, next_state, terminal_state)
                running_reward += reward.item()
                transition_ = Transition(state=state, action=action,
                                         reward=reward, next_state=next_state)
                replay_memory_.push(*transition_)
                if printing_while_training:
                    print('{}: Finished after {} iterations with reward {} '
                          'in state {} starting from {}'
                          .format(i + 1, iteration + 1, running_reward,
                                  next_state.data.numpy(),
                                  starting_state.data.numpy()))
                break
            # abort if illegal state
            elif torch.eq(next_state.abs(), 15).any() or iteration == 99:
                reward = compute_reward_2(state, next_state, terminal_state)
                running_reward += reward
                transition_ = Transition(state=state, action=action,
                                         reward=reward, next_state=next_state)
                replay_memory_.push(*transition_)
                if printing_while_training:
                    print('{}: Aborted after {} iterations with reward {} '
                          'in state {} starting from {}'
                          .format(i + 1, iteration + 1, running_reward,
                                  next_state.data.numpy(),
                                  starting_state.data.numpy()))
                break

            # compute immediate reward
            reward = compute_reward_2(state, next_state, terminal_state)
            # save it - only for logging purposes
            running_reward += reward.item()

            # construct transition
            transition_ = Transition(state=state, action=action, reward=reward,
                                     next_state=next_state)

            # update model
            update(transition_, replay_memory_, batch_size, gamma)
            # perform soft updates
            update_weights(actor_target, actor_trained, tau)
            update_weights(critic_target, critic_trained, tau)

            state = next_state
            iteration += 1
    print('Ended after: {}'.format(datetime.datetime.now() - starting_time))

    print('######################## Testing ########################')
    starting_time = datetime.datetime.now()
    test_states = [torch.tensor([i, j], device=device, dtype=torch.int)
                   for i in range(-15, 14) for j in range(-15, 14)]
    finished = 0
    aborted = 0
    aborted_reward = []
    finished_reward = []

    for starting_state in test_states:
        state = starting_state.clone()
        terminal_state = torch.tensor([0, 0], device=device, dtype=torch.int)
        iteration = 0
        reward = 0.

        while True:
            action = torch.argmax(actor_target(state.float())).view(1).int()
            next_state = compute_next_state(action, state)

            if torch.eq(next_state, terminal_state).all():
                reward += compute_reward_2(state, next_state,
                                           terminal_state)
                finished_reward.append(reward.item())
                if printing_while_testing:
                    print('{}: Finished after {} iterations with reward {} '
                          'in state {} starting from {}'
                          .format(starting_state.data.numpy(), iteration + 1,
                                  reward.item(), next_state.data.numpy(),
                                  starting_state.data.numpy()))
                finished += 1
                break
            elif torch.eq(next_state.abs(), 15).any():
                reward += compute_reward_2(state, next_state,
                                           terminal_state)
                aborted_reward.append(reward.item())
                if printing_while_testing:
                    print('{}: Aborted after {} iterations with reward {} '
                          'in state {} starting from {}'
                          .format(starting_state.data.numpy(), iteration + 1,
                                  reward.item(), next_state.data.numpy(),
                                  starting_state.data.numpy()))
                aborted += 1
                break
            elif iteration > 500:
                if printing_while_testing:
                    print('Aborting due to more than 500 iterations! '
                          'Started from {}'.format(
                        starting_state.data.numpy()))
                aborted += 1
                break
            reward += compute_reward_2(state, next_state, terminal_state)
            state = next_state
            iteration += 1

    print('Ended after: {}'.format(datetime.datetime.now() - starting_time))
    print('Finished: {}, aborted: {}'.format(finished, aborted))
    print('Reward mean finished: {}, aborted: {}'
          .format(np.mean(finished_reward), np.mean(aborted_reward)))

I already tried using an other reward function but it did not have any effect.. .

Additionally I tried using some less aggressive exploration and optim.SGD instead of optim.RMSprop - both had no effect.

Output:
This might now be the direct answer or a recipe for your code to work, but I have some initial worries that might help you debug the code.

The biggest issue is, I believe, that you perform several conversions to datatypes that are not tensors. For example, you call your combine_tensors() functions couple of times, and it converts given tensors to numpy() and creates new tensor when returning the value. Other time you call your networks to perform the forward pass, and give them tensors converted with float() function as the argument. There are calls to int() on tensors aswell. All these calls cause losing the tensor's operation graph that is used for computing the gradient on the backward() call. This is described in PyTorch documentation and should be understood before writing RL algorithms in this framework. It is essential to work on the tensors for the whole time you are in the train function - from the time you convert the experience batch to tensors, to the time you call the backward functions.

This alone still does not guarantee that the learning will be performed in a correct way. For example, when you use the target networks to estimate the loss for critic, you should detach the results to prevent the gradient computations in target networks (although, if you use optimizer, and you register only the critic parameters, it is more of a performance issue as step() call won't update target network's parameters).

When both issues will be resolved in your code, you might observe more correct behaviour. My additional comment here is that I don't really understand parts of your code and I think it is not a correct DDPG implementation (i. e. you use argmax() on actor network output and provide this to critic network and this does not look like the correct way).

I'd advice you to take a step back and get a bit more understanding of PyTorch framework and ideas, as well as look for some baseline implementations of DDPG to make sure you know how to perform the calculations step by step.


Instruction: Pytorch gradients exist but weights not updating

Input:
So, I have a deep convolutional network with an lstm layer, and after the ltsm layer it splits off to compute two different functions (using two different linear layers) whose results are then added together to form the final network output.

When I compute the loss of the network so that I can have it compute the gradients and update the weights, I have it do a few operations and then have it compute the loss between the derived value and the calculated target value.

def update(output, target):
    # target output is calculated outside the function
    # operations on output
    loss(output, target).backward()
    self.optimizer.step()

The network has some loss (sometimes in a very small order of magnitude, but sometimes also on higher orders of magnitude), for example a few of the losses:

tensor(1.00000e-04 *
   5.7420)
tensor(2.7190)
tensor(0.9684)

It also has gradients as calculated here:

for param in self.parameters():
    print(param.grad.data.sum())

Which outputs:

tensor(1.00000e-03 *
   1.9996)
tensor(1.00000e-03 *
   2.6101)
tensor(1.00000e-02 *
   -1.3879)
tensor(1.00000e-03 *
   -4.5834)
tensor(1.00000e-02 *
   2.1762)
tensor(1.00000e-03 *
   3.6246)
tensor(1.00000e-03 *
   6.6234)
tensor(1.00000e-02 *
   2.9373)
tensor(1.00000e-02 *
   1.2680)
tensor(1.00000e-03 *
   1.8791)
tensor(1.00000e-02 *
   1.7322)
tensor(1.00000e-02 *
   1.7322)
tensor(0.)
tensor(0.)
tensor(1.00000e-03 *
   -6.7885)
tensor(1.00000e-02 *
   9.7793)

And:

tensor(2.4620)
tensor(0.9544)
tensor(-26.2465)
tensor(0.2280)
tensor(-219.2602)
tensor(-2.7870)
tensor(-50.8203)
tensor(3.2548)
tensor(19.6163)
tensor(-18.6029)
tensor(3.8564)
tensor(3.8564)
tensor(0.)
tensor(0.)
tensor(0.8040)
tensor(-0.1157)

But when I compare the weight before and after running the optimizer, I get the result that the weights are equal to each other.

Code to see if weights change:

before = list(neuralnet.parameters())
neuralnet.update()
after = list(neuralnet.parameters())
for i in range(len(before)):
    print(torch.equal(before[i].data, after[i].data))

The above returns True for each iteration.

Output: https://discuss.pytorch.org/t/gradients-exist-but-weights-not-updating/20484/2?u=wr01 has the answer I sought. The problem was that neuralnet.parameters() does not clone the list of parameters, so when I was updating the weights, the weights were updating in the before variable.


Instruction: Custom weight initialization in PyTorch

Input:
What would be the right way to implement a custom weight initialization method in PyTorch?

I believe I can't directly add any method to 'torch.nn.init` but wish to initialize my model's weights with my own proprietary method.

Output:
You can define a method to initialize the weights according to each layer:

def weights_init(m):
    classname = m.__class__.__name__

    if classname.find('Conv2d') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

And then just apply it to your network:

model = create_your_model()
model.apply(weights_init)


Instruction: How do I create a normal distribution in pytorch?

Input: I want to create a random normal distribution with a given mean and std.

Output:
You can easily use torch.Tensor.normal_() method.

Let's create a matrix Z (a 1d tensor) of dimension 1 × 5, filled with random elements samples from the normal distribution parameterized by mean = 4 and std = 0.5.

torch.empty(5).normal_(mean=4,std=0.5)

Result:

tensor([4.1450, 4.0104, 4.0228, 4.4689, 3.7810])


Instruction: Difference between tensor.permute and tensor.view in PyTorch?

Input:
What is the difference between tensor.permute() and tensor.view()?

They seem to do the same thing.

Output:
Input
In [12]: aten = torch.tensor([[1, 2, 3], [4, 5, 6]])

In [13]: aten
Out[13]:
tensor([[ 1,  2,  3],
        [ 4,  5,  6]])

In [14]: aten.shape
Out[14]: torch.Size([2, 3])

torch.view() reshapes the tensor to a different but compatible shape. For example, our input tensor aten has the shape (2, 3). This can be viewed as tensors of shapes (6, 1), (1, 6) etc.,
# reshaping (or viewing) 2x3 matrix as a column vector of shape 6x1
In [15]: aten.view(6, -1)
Out[15]:
tensor([[ 1],
        [ 2],
        [ 3],
        [ 4],
        [ 5],
        [ 6]])

In [16]: aten.view(6, -1).shape
Out[16]: torch.Size([6, 1])

Alternatively, it can also be reshaped or viewed as a row vector of shape (1, 6) as in:
In [19]: aten.view(-1, 6)
Out[19]: tensor([[ 1,  2,  3,  4,  5,  6]])

In [20]: aten.view(-1, 6).shape
Out[20]: torch.Size([1, 6])

Whereas tensor.permute() is only used to swap the axes. The below example will make things clear:
In [39]: aten
Out[39]:
tensor([[ 1,  2,  3],
        [ 4,  5,  6]])

In [40]: aten.shape
Out[40]: torch.Size([2, 3])

# swapping the axes/dimensions 0 and 1
In [41]: aten.permute(1, 0)
Out[41]:
tensor([[ 1,  4],
        [ 2,  5],
        [ 3,  6]])

# since we permute the axes/dims, the shape changed from (2, 3) => (3, 2)
In [42]: aten.permute(1, 0).shape
Out[42]: torch.Size([3, 2])

You can also use negative indexing to do the same thing as in:
In [45]: aten.permute(-1, 0)
Out[45]:
tensor([[ 1,  4],
        [ 2,  5],
        [ 3,  6]])

In [46]: aten.permute(-1, 0).shape
Out[46]: torch.Size([3, 2])


Instruction: Is there a bug with PyTorch training for large batch sizes, or with this script?

Input:
I am following this PyTorch tutorial by Joshua L. Mitchell.  The grand finale of the tutorial is the following PyTorch training script.  One element, the batch size, I have parameterized in the first line of the script, which I run in a newly started Jupyter notebook.  The key parameter in question is BIGGER_BATCH, initially set to 4:

BIGGER_BATCH=4

import numpy as np
import torch # Tensor Package (for use on GPU)
import torch.nn as nn ## Neural Network package
import torch.optim as optim # Optimization package
import torchvision # for dealing with vision data
import torchvision.transforms as transforms # for modifying vision data to run it through models
from torch.autograd import Variable # for computational graph
import torch.nn.functional as F # Non-linearities package
import matplotlib.pyplot as plt # for plotting

def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

transform = transforms.Compose( # we're going to use this to transform our data to make each sample more uniform
   [
    transforms.ToTensor(), # converts each sample from a (0-255, 0-255, 0-255) PIL Image format to a (0-1, 0-1, 0-1) FloatTensor format
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) # for each of the 3 channels of the image, subtract mean 0.5 and divide by stdev 0.5
   ]) # the normalization makes each SGD iteration more stable and overall makes convergence easier

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform) # this is all we need to get/wrangle the dataset!

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)

trainloader = torch.utils.data.DataLoader(trainset, batch_size=BIGGER_BATCH,
                                          shuffle=False)
testloader = torch.utils.data.DataLoader(testset, batch_size=BIGGER_BATCH,
                                         shuffle=False)

classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # each image can have 1 of 10 labels

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 10, 5) # Let's add more feature maps - that might help
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(10, 20, 5) # And another conv layer with even more feature maps
        self.fc1 = nn.Linear(20 * 5 * 5, 120) # and finally, adjusting our first linear layer's input to our previous output
        self.fc2 = nn.Linear(120, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x) # we're changing our nonlinearity / activation function from sigmoid to ReLU for a slight speedup
        x = self.pool(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = self.pool(x) # after this pooling layer, we're down to a torch.Size([4, 20, 5, 5]) tensor.
        x = x.view(-1, 20 * 5 * 5) # so let's adjust our tensor again.
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        x = F.relu(x)
        return x

net = Net().cuda()

NUMBER_OF_EPOCHS = 25
LEARNING_RATE = 1e-2
loss_function = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=LEARNING_RATE)

for epoch in range(NUMBER_OF_EPOCHS):
    train_loader_iter = iter(trainloader)
    for batch_idx, (inputs, labels) in enumerate(train_loader_iter):
        net.zero_grad()
        inputs, labels = Variable(inputs.float().cuda()), Variable(labels.cuda())
        output = net(inputs)
        loss = loss_function(output, labels)
        loss.backward()
        optimizer.step()
    if epoch % 5 is 0:
        print("Iteration: " + str(epoch + 1))

dataiter = iter(testloader)
images, labels = dataiter.next()

imshow(torchvision.utils.make_grid(images[0:4]))

outputs = net(Variable(images.cuda()))
_, predicted = torch.max(outputs.data, 1)
print('Predicted: ', ' '.join('%5s' % classes[predicted[j]]
                              for j in range(4)))
print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))

correct = 0
total = 0
for data in testloader:
    images, labels = data
    labels = labels.cuda()
    outputs = net(Variable(images.cuda()))
    _, predicted = torch.max(outputs.data, 1)
    total += labels.size(0)
    correct += (predicted == labels).sum()
print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

This gets a good, expected result of 58% accuracy:

Predicted:    cat  ship  ship plane
GroundTruth:    cat  ship  ship plane
Accuracy of the network on the 10000 test images: 58 %

Now if I change the first line of the above script to

BIGGER_BATCH=4096

and I restart the kernel and run the script I get 19% accuracy, consistently:

Predicted:    car  ship  ship  ship
GroundTruth:    cat  ship  ship plane
Accuracy of the network on the 10000 test images: 19 %

Note that I am not shuffling the inputs, so I can't attribute this change to the order of inputs in the training:

trainloader = torch.utils.data.DataLoader(trainset, batch_size=BIGGER_BATCH,
                                          shuffle=False)
testloader = torch.utils.data.DataLoader(testset, batch_size=BIGGER_BATCH,
                                         shuffle=False)

What accounts for the huge drop in accuracy when I increase the batch size?  Is there something wrong in the script, or something wrong in PyTorch, or something else I'm not thinking of?

Output: Sorry I just realized this was a dumb question.  I am doing far fewer updates -- by a factor of 1,024.  That's why the accuracy is much lower.  I could adjust the learning rate but there is clearly a tradeoff between batch size and learning rate that I am just learning about now.


Instruction: Import System Modules in a Conda Environment

Input:
I have installed caffe and pytorch0.3 in my system path(environment), and there is a project which only works under caffe and history version of pytorch0.2. To solve that I install pytorch0.2 in a Conda environment and I wonder if there is any way can save me from installing a caffe again in this conda environment.

In other words, can I use pytorch0.2 in this Conda environment meanwhile import the system caffe? and how?

Output:
Activate your environment and try the following

conda install package-name --offline

Also, in case you wish to clone the root or some other environment to a conda environment, you can use -- clone. For instance, when you wish to clone the root -

conda create -n pytorch_02 --clone root


Instruction: Pytorch: Dropout Layers and Packed Sequences

Input:
(PyTorch 0.4.0)

How does one apply a manual dropout layer to a packed sequence (specifically in an LSTM on a GPU)? Passing the packed sequence (which comes from the lstm layer) directly does not work, as the dropout layer doesn’t know quite what to do with it and returns something not a packed sequence. Passing the data of the packed sequence seems like it should work, but results in the attribute error shown below the code sample.

    def __init__ (self, ....):
        super(Model1, self).__init__()
        ....
        self.drop = torch.nn.Dropout(p=0.5, inplace=False)

    def forward(self, inputs, lengths):
        pack1 = nn.utils.rnn.pack_padded_sequence(inputs, lengths, batch_first=True)
        out1, self.hidden1 = self.lstm1(pack1, (self.hidden1[0].detach(), self.hidden1[1].detach()))
        out1.data = self.drop(out1.data)

This results in:

 AttributeError: can't set attribute

Perversely, I can make this an inplace operation (again, on the data directly, not the full packed sequence) and it technically works (i.e., it runs) on the CPU, but gives a warning on the GPU that the inplace operation is modifying a needed gradient.  This gives me no confidence that the CPU version works correctly (does it?  Is it missing a warning?  Would not be the first time that I caught PyTorch silently chugging along doing something it should flag a warning for) and in any case GPU support is vital

So:

Are the different behaviors between CPU and GPU expected?
What is the overall correct way to do this on a GPU?
What is the overall correct way to do this on a CPU?

Output:
You can use pad_packed_sequence:

def forward(self, inputs, lengths):
        pack1 = nn.utils.rnn.pack_padded_sequence(inputs, lengths, batch_first=True)
        out1, self.hidden1 = self.lstm1(pack1, (self.hidden1[0].detach(), self.hidden1[1].detach()))
        out1, _ = nn.utils.rnn.pad_packed_sequence(out1, batch_first=True)
        out1 = self.drop(out1)


Instruction: Neural networks pytorch

Input:
I am very new in pytorch and implementing my own network of image classifier. However I see for each epoch training accuracy is very good but validation accuracy is 0.i noted till 5th epoch. I am using Adam optimizer and have learning rate .001. also resampling the whole data set after each epoch into training n validation set. Please help where I am going wrong.

Here is my code:

### where is data?
data_dir_train = '/home/sup/PycharmProjects/deep_learning/CNN_Data/training_set'
data_dir_test = '/home/sup/PycharmProjects/deep_learning/CNN_Data/test_set'

# Define your batch_size
batch_size = 64

allData = datasets.ImageFolder(root=data_dir_train,transform=transformArr)

# We need to further split our training dataset into training and validation sets.
def split_train_validation():
    # Define the indices
    num_train = len(allData)
    indices = list(range(num_train)) # start with all the indices in training set
    split = int(np.floor(0.2 * num_train)) # define the split size
    #train_idx, valid_idx = indices[split:], indices[:split]

    # Random, non-contiguous split
    validation_idx = np.random.choice(indices, size=split, replace=False)
    train_idx = list(set(indices) - set(validation_idx))
    # define our samplers -- we use a SubsetRandomSampler because it will return
    # a random subset of the split defined by the given indices without replacement
    train_sampler = SubsetRandomSampler(train_idx)
    validation_sampler = SubsetRandomSampler(validation_idx)

    #train_loader = DataLoader(allData,batch_size=batch_size,sampler=train_sampler,shuffle=False,num_workers=4)
    #validation_loader = DataLoader(dataset=allData,batch_size=1, sampler=validation_sampler)

    return (train_sampler,validation_sampler)

Training

from torch.optim import Adam
import torch
import createNN
import torch.nn as nn
import loadData as ld
from torch.autograd import  Variable
from torch.utils.data import DataLoader

# check if cuda - GPU support available
cuda = torch.cuda.is_available()

#create model, optimizer and loss function
model = createNN.ConvNet(class_num=2)
optimizer = Adam(model.parameters(),lr=.001,weight_decay=.0001)
loss_func = nn.CrossEntropyLoss()

if cuda:
    model.cuda()

# function to save model
def save_model(epoch):
    torch.save(model.load_state_dict(),'imageClassifier_{}.model'.format(epoch))
    print('saved model at epoch',epoch)

def exp_lr_scheduler (  epoch , init_lr = args.lr, weight_decay = args.weight_decay, lr_decay_epoch = cf.lr_decay_epoch):
 lr = init_lr * ( 0.5 ** (epoch // lr_decay_epoch))

def train(num_epochs):
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('\n\nEpoch {}'.format(epoch))
        train_sampler, validation_sampler = ld.split_train_validation()
        train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
        validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
        model.train()
        acc = 0.0
        loss = 0.0
        total = 0
        # train model with training data
        for i,(images,labels) in enumerate(train_loader):
            # if cuda then move to GPU
            if cuda:
                images = images.cuda()
                labels = labels.cuda()
            # Variable class wraps a tensor and we can calculate grad
            images = Variable(images)
            labels = Variable(labels)
            # reset accumulated gradients for each batch
            optimizer.zero_grad()
            # pass images to model which returns preiction
            output = model(images)
            #calculate the loss based on prediction and actual
            loss = loss_func(output,labels)
            # backpropagate the loss and compute gradient
            loss.backward()
            # update weights as per the computed gradients
            optimizer.step()

            # prediction class
            predVal , predClass = torch.max(output.data, 1)
            acc += torch.sum(predClass == labels.data)
            loss += loss.cpu().data[0]
            total += labels.size(0)
        # print the statistics
        train_acc = acc/total
        train_loss = loss / total
        print('Mean train acc = {} over epoch = {}'.format(epoch,acc))
        print('Mean train loss = {} over epoch = {}'.format(epoch, loss))

        # Valid model with validataion data
        model.eval()
        acc = 0.0
        loss = 0.0
        total = 0
        for i,(images,labels) in enumerate(validation_loader):
            # if cuda then move to GPU
            if cuda:
                images = images.cuda()
                labels = labels.cuda()
            # Variable class wraps a tensor and we can calculate grad
            images = Variable(images)
            labels = Variable(labels)
            # reset accumulated gradients for each batch
            optimizer.zero_grad()
            # pass images to model which returns preiction
            output = model(images)
            #calculate the loss based on prediction and actual
            loss = loss_func(output,labels)
            # backpropagate the loss and compute gradient
            loss.backward()
            # update weights as per the computed gradients
            optimizer.step()

            # prediction class
            predVal, predClass = torch.max(output.data, 1)
            acc += torch.sum(predClass == labels.data)
            loss += loss.cpu().data[0]
            total += labels.size(0)
        # print the statistics
        valid_acc = acc / total
        valid_loss = loss / total
        print('Mean train acc = {} over epoch = {}'.format(epoch, valid_acc))
        print('Mean train loss = {} over epoch = {}'.format(epoch, valid_loss))

        if(best_acc<valid_acc):
            best_acc = valid_acc
            save_model(epoch)

        # at 30th epoch we save the model
        if (epoch == 30):
            save_model(epoch)

train(20)

Output:
I think you did not take into account that acc += torch.sum(predClass == labels.data) returns a tensor instead of a float value. Depending on the version of pytorch you are using I think you should change it to:

acc += torch.sum(predClass == labels.data).cpu().data[0] #pytorch 0.3
acc += torch.sum(predClass == labels.data).item() #pytorch 0.4

Although your code seems to be working for old pytorch version, I would recommend you to upgrade to the 0.4 version.

Also, I mentioned other problems/typos in your code.

You are loading the dataset for every epoch.

for epoch in range(num_epochs):
    print('\n\nEpoch {}'.format(epoch))
    train_sampler, validation_sampler = ld.split_train_validation()
    train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
    validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
    ...

That should not happen, it should be enough loading it once

train_sampler, validation_sampler = ld.split_train_validation()
train_loader = DataLoader(ld.allData, batch_size=30, sampler=train_sampler, shuffle=False)
validation_loader = DataLoader(dataset=ld.allData, batch_size=1, sampler=validation_sampler)
for epoch in range(num_epochs):
    print('\n\nEpoch {}'.format(epoch))
    ...

In the training part you have (this does not happen in the validation):

train_acc = acc/total
train_loss = loss / total
print('Mean train acc = {} over epoch = {}'.format(epoch,acc))
print('Mean train loss = {} over epoch = {}'.format(epoch, loss))

Where you are printing acc instead of train_acc

Also, in the validation part I mentioned that you are printing print('Mean train acc = {} over epoch = {}'.format(epoch, valid_acc)) when it should be something like 'Mean val acc'.

Changing this lines of code, using a standard model I created and CIFAR dataset the training seems to converge, accuracy increases at every epoch while mean loss value decreases.

I Hope I could help you!


Instruction: Loss to minimize the overlap of two collections/sets

Input:
I wonder if there is a loss function can measure the overlap of two collections/sets (order doesn't matter).
E.g. the groud truth is a set [a, b, c] and my model prediction is a set [b, e, f], the overlap is [b]. My goal is to maximize the overlap of my prediction.
Do we have a loss function that can measure the size of overlap that I can minimize (negative of) the metric as a result I can maximize the overlap.
(I know one solution may follow the REIFORCE learning that treat the overlap as a reward for each data sample and use the reward to weight the loss, but do we have another solution)
Thank you.

Output:
As noted by P-Gn, the problem with such coefficients is their differentiability.
However, it is possible to define similar measures to these coefficients that are differentiable. IOU (intersection over union), as proposed by Prune is a good measure. For deep learning tasks more popular is the similar Dice coefficient:

2 * len(A intersect B)/(len(A)+ len(B))

which ranges between 0 if no overlap and 1 for identical sets.
For binary vectors this can be formulated as

2 * abs(a.b)/(a**2 + b**2)

where the vectors are a one-hot encoded representation of the set.

Now, if your last layer in your neural network has a softmax activation (like when you use cross entropy) you can interpret the output as the probabilities of that particular element belonging to your predicted set. The previous formula still is a good measure for the intersection between your sets but stays differentiable. The so called Dice loss (1 - dice coefficient) was introduced first in this paper where you can read more about it.


Instruction: GPU program failed to execute : cublas runtime error

Input:
I am trying to train a network via pytorch on CUDA enabled GeForce GTX 1070 gpu. I don't understand the error nor have I found any similar problem anywhere. I don't know if its cuda's issue or something in my code.

Traceback (most recent call last):
  File "main.py", line 497, in <module>
    main()
  File "main.py", line 167, in main
    train(train_loader, model, criterion, optimizer, epoch, normalizer)
  File "main.py", line 244, in train
    output = model(*input_var)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "C:\Users\1546544\Desktop\ML\model.py", line 147, in forward
    atom_fea = conv_func(atom_fea, nbr_fea, nbr_fea_idx)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "C:\Users\1546544\Desktop\ML\model.py", line 66, in forward
    total_gated_fea = self.fc_full(total_nbr_fea)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py", line 357, in __call__
    result = self.forward(*input, **kwargs)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\linear.py", line 55, in forward
    return F.linear(input, self.weight, self.bias)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\functional.py", line 837, in linear
    output = input.matmul(weight.t())
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\autograd\variable.py", line 386, in matmul
    return torch.matmul(self, other)
  File "C:\ProgramData\Anaconda3\lib\site-packages\torch\functional.py", line 192, in matmul
    output = torch.mm(tensor1, tensor2)
RuntimeError: cublas runtime error : the GPU program failed to execute at C:/Anaconda2/conda-bld/pytorch_1519496000060/work/torch/lib/THC/THCBlas.cu:247

Output:
I faced the same problem.

I fixed this problem by dataset label correction.
I mean, training label was incorrect for my dataset. That's why it was failed during backward() pass.

So, checking the expected label after loading it from disk/database might be helpful.


Instruction: 2d array as index in Pytorch

Input:
I want to ‘grow’ a matrix using a set of rules.

Example of rules:

0->[[1,1,1],[0,0,0],[2,2,2]],
1->[[2,2,2],[2,2,2],[2,2,2]],
2->[[0,0,0],[0,0,0],[0,0,0]]

Example of growing a matrix:

[[0]]->[[1,1,1],[0,0,0],[2,2,2]]->
[[2,2,2,2,2,2,2,2,2],[2,2,2,2,2,2,2,2,2],[2,2,2,2,2,2,2,2,2],
[1,1,1,1,1,1,1,1,1],[0,0,0,0,0,0,0,0,0],[2,2,2,2,2,2,2,2,2],
[0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0]]

This is the code I’ve been trying to get to work in Pytorch

rules = np.random.randint(256,size=(10,256,3,3,3))
rules_tensor = torch.randint(256,size=(10,
            256, 3, 3, 3),
            dtype=torch.uint8, device = torch.device('cuda'))

rules = rules[0]
rules_tensor = rules_tensor[0]

seed = np.array([[128]])
seed_tensor = seed_tensor = torch.cuda.ByteTensor([[128]])

decode = np.empty((3**3, 3**3, 3))
decode_tensor = torch.empty((3**3,
                3**3, 3), dtype=torch.uint8,
                device = torch.device('cuda'))

for i in range(3):
    grow = seed
    grow_tensor = seed_tensor
    for j in range(1,4):
        grow = rules[grow,:,:,i].reshape(3**j,-1)
        grow_tensor = rules_tensor[grow_tensor,:,:,i].reshape(3**j,-1)

    decode[..., i] = grow
    decode_tensor[..., i] = grow_tensor

I can’t seem to select indices the same way as in Numpy in this line:

grow = rules[grow,:,:,i].reshape(3**j,-1)

Is there a way to do the following in Pytorch?

Output:
You could consider using torch.index_select(), flattening your index tensor before reshaping the result:

Code:

import torch
import numpy as np

rules_np = np.array([
    [[1,1,1],[0,0,0],[2,2,2]],  # for value 0
    [[2,2,2],[2,2,2],[2,2,2]],  # for value 1
    [[0,0,0],[0,0,0],[0,0,0]]]) # for value 2, etc.
rules = torch.from_numpy(rules_np).long()
rule_shape = rules[0].shape

seed = torch.zeros(1).long()
num_growth = 2
print("Seed:")
print(seed)

grow = seed
for i in range(num_growth):
    grow = (torch.index_select(rules, 0, grow.view(-1))
            .view(grow.shape + rule_shape)
            .squeeze())
    print("Growth #{}:".format(i))
    print(grow)

Log:

Seed:
tensor([ 0])
Growth #0:
tensor([[ 1,  1,  1], [ 0,  0,  0], [ 2,  2,  2]])
Growth #1:
tensor([[[[ 2,  2,  2], [ 2,  2,  2], [ 2,  2,  2]],
         [[ 2,  2,  2], [ 2,  2,  2], [ 2,  2,  2]],
         [[ 2,  2,  2], [ 2,  2,  2], [ 2,  2,  2]]],

        [[[ 1,  1,  1], [ 0,  0,  0], [ 2,  2,  2]],
         [[ 1,  1,  1], [ 0,  0,  0], [ 2,  2,  2]],
         [[ 1,  1,  1], [ 0,  0,  0], [ 2,  2,  2]]],

        [[[ 0,  0,  0], [ 0,  0,  0], [ 0,  0,  0]],
         [[ 0,  0,  0], [ 0,  0,  0], [ 0,  0,  0]],
         [[ 0,  0,  0], [ 0,  0,  0], [ 0,  0,  0]]]])


Instruction: How to utilize all GPUs when dealing with pytorch code?

Input:
I have 2 GPUs and when I am working with pytorch code, only one GPU is used. I tried CUDA_VISIBLE_DEVICES=0,1 python xxx.py, but occurs

  'CUDA_VISIBLE_DEVICES: command not found'

problems. I have also tried to add the following lines in object py file:

 import os
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

but still only one GPU is utilized.

Output:
You need to parallelize the training data to each GPU seperatly. Data Parallelism is implemented using torch.nn.DataParallel. An example from the pytorch documentation:

import torch
import torch.nn as nn

class DataParallelModel(nn.Module):

    def __init__(self):
        super().__init__()
        self.block1 = nn.Linear(10, 20)

        # wrap block2 in DataParallel
        self.block2 = nn.Linear(20, 20)
        self.block2 = nn.DataParallel(self.block2)

        self.block3 = nn.Linear(20, 20)

    def forward(self, x):
        x = self.block1(x)
        x = self.block2(x)
        x = self.block3(x)
        return x


Instruction: Which function in Tensorflow is similar to expand_as in Pytorch

Input:
I am trying to adapt some code from Pytorch to Tensorflow, though I'm not so familiar with the latter.

I use expand_as in my old code. Is there any similar function which can be used in Tensorflow?

Output: You are looking for tf.broadcast_to which has been introduced in TF 1.9 (initially under the name tf.contrib.framework.broadcast_to).


Instruction: Pytorch, `backward` RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed

Input:
I'm implementing DDPG with PyTorch (0.4) and got stuck backproping the loss.
So, first my Code performing the update:
def update_nets(self, transitions):
    """
    Performs one update step
    :param transitions: list of sampled transitions
    """
    # get batches
    batch = transition(*zip(*transitions))
    states = torch.stack(batch.state)
    actions = torch.stack(batch.action)
    next_states = torch.stack(batch.next_state)
    rewards = torch.stack(batch.reward)

    # zero gradients
    self._critic.zero_grad()

    # compute critic's loss
    y = rewards.view(-1, 1) + self._gamma * \
        self.critic_target(next_states, self.actor_target(next_states))

    loss_critic = F.mse_loss(y, self._critic(states, actions),
                             size_average=True)

    # backpropagte it
    loss_critic.backward()
    self._optim_critic.step()

    # zero gradients
    self._actor.zero_grad()

    # compute actor's loss
    loss_actor = ((-1.) * self._critic(states, self._actor(states))).mean()

    # backpropagate it
    loss_actor.backward()
    self._optim_actor.step()

    # do soft updates
    self.perform_soft_update(self.actor_target, self._actor)
    self.perform_soft_update(self.critic_target, self._critic)

Where self._actor, self._crtic, self.actor_target and self.critic_target are Nets.
If I run this, I get the following error in the second iteration:
RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.

at

line 221, in update_nets
    loss_critic.backward()
line 93, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
line 89, in backward
    allow_unreachable=True)  # allow_unreachable flag

and I don't know what is causing it.
What I know till now is, that the loss_critic.backward() call causes the error.
I already debugged loss_critic - it got a valid value.
If I replace the loss computation with a simple
loss_critic = torch.tensor(1., device=self._device, dtype=torch.float, requires_grad=True)

Tensor containing the value 1 it works fine.
Also, I already checked that I'm not saving some results which could cause the error.
Additionally updating the actor with the loss_actor doesn't cause any problems.
Does anyone know what is going wrong here?
Thanks!
Update
I replaced
    # zero gradients
    self._critic.zero_grad()

and
    # zero gradients
    self._actor.zero_grad()

with
    # zero gradients
    self._critic.zero_grad()
    self._actor.zero_grad()
    self.critic_target.zero_grad()
    self.actor_target.zero_grad()

(both calls) but it is still failing with the same error.
Additionally, the code performing the update at the end of one iteration
def perform_soft_update(self, target, trained):
    """
    Preforms the soft update
    :param target: Net to be updated
    :param trained: Trained net - used for update
    """
    for param_target, param_trained in \
            zip(target.parameters(), trained.parameters()):
        param_target.data.copy_(
            param_target.data * (
                    1.0 - self._tau) + param_trained * self._tau
        )

Output:
I found the solution.
I saved tensors in my replay_buffer, for training purposes, which I used in every iteration resulting in the code snippet:
    # get batches
    batch = transition(*zip(*transitions))
    states = torch.stack(batch.state)
    actions = torch.stack(batch.action)
    next_states = torch.stack(batch.next_state)
    rewards = torch.stack(batch.reward)

This "saving" of tensors is the cause of the problem. So I changed my code to save only the data (tensor.data.numpy().tolist()) and only put it into a tensor when I need it.
More detailed:
In DDPG I evaluate the policy every iteration and do one learning step with a batch.
Now I'm saving the evaluation in the replay buffer via:
action = self.action(state)
...
self.replay_buffer.push(state.data.numpy().tolist(), action.data.numpy().tolist(), ...)

And used it like:
batch = transition(*zip(*transitions))
states = self.to_tensor(batch.state)
actions = self.to_tensor(batch.action)
next_states = self.to_tensor(batch.next_state)
rewards = self.to_tensor(batch.reward)


Instruction: Is column selection in pytorch differentiable?

Input:
Is column selection in Pytorch differentiable? for eg if I want to select a single column from each row to make a new row X 1 array and then backdrop using this new array, will the backdrop work properly?

qvalues = qvalues[range(5),[0,1,0,1,0]]

if element selection is done as shown above from a 5*2 tensor?

Output:
I think it is. Let me make an example with code.

First we create the qvalues tensor and we say we want to compute its gradients

qvalues = torch.rand((5, 5), requires_grad=True)

Now we create the tensor to index it and obtain a 5x2 tensor as its result (I think this is the same selection you wanted to perform with qvalues[range(5),[0,1,0,1,0]]):

y = torch.LongTensor([1, 3])
new_qvalues = qvalues[:, y]

We see that the slice new_qvalues of the original qvalues will compute the gradient

print(new_qvalues.requires_grad) # True

Now we perform our mathematical operations. In this example code, I am doing the square of new_qvalues because we know that its gradient (derivative) will be 2 * new_qvalues.

qvalues_a = new_qvalues ** 2

Now, we have to compute the gradients of qvalues_a. We set retain_graph=True to store the .grad of each tensor and avoid freeing the buffers on the backward pass.

qvalues_a.backward(torch.ones(new_qvalues.shape), retain_graph=True)

Now, we can go back to the original qvalues and see if the gradients have been calculated

print(qvalues)
print(qvalues.grad)

# result of the print statemets

#tensor([[ 0.9677,  0.4303,  0.2036,  0.3870,  0.6085],
#    [ 0.8876,  0.8695,  0.2028,  0.3283,  0.1560],
#    [ 0.1764,  0.4718,  0.5418,  0.5167,  0.6200],
#    [ 0.7610,  0.9322,  0.5584,  0.5589,  0.8901],
#    [ 0.8146,  0.7296,  0.8036,  0.5277,  0.5754]])

#tensor([[ 0.0000,  0.8606,  0.0000,  0.7739,  0.0000],
#    [ 0.0000,  1.7390,  0.0000,  0.6567,  0.0000],
#    [ 0.0000,  0.9435,  0.0000,  1.0334,  0.0000],
#    [ 0.0000,  1.8645,  0.0000,  1.1178,  0.0000],
#    [ 0.0000,  1.4592,  0.0000,  1.0554,  0.0000]])

We can observe how the gradients have been computed only in the selected indexes. To be sure about it we create some fast test by comparing that the value of qvalues.grad for the selected slice is equal to the derivate 2 * new_qvalues.

assert torch.equal(qvalues.grad[:, y], 2 * new_qvalues)

And it does not throw any error, so I would assume that you can get the gradient of the slice.


Instruction: Calculate covariance matrix for complex data in two channels (no complex data type)

Input:
I have complex-valued data given in 2 channels of a matrix (one is the real, one the imaginary part, so the matrix dimensions are (height, width, 2), since Pytorch does not have native complex data types. I now want to calculate the covariance matrix. The stripped-down numpy calculation adapted for Pytorch is this:

def cov(m, y=None):

    if m.ndimension() > 2:
        raise ValueError("m has more than 2 dimensions")

    if y.ndimension() > 2:
        raise ValueError('y has more than 2 dimensions')

    X = m
    if X.shape[0] == 0:
        return torch.tensor([]).reshape(0, 0)
    if y is not None:
        X = torch.cat((X, y), dim=0)

    ddof = 1

    avg = torch.mean(X, dim=1)

    fact = X.shape[1] - ddof

    if fact <= 0:
        import warnings
        warnings.warn("Degrees of freedom <= 0 for slice",
                      RuntimeWarning, stacklevel=2)
        fact = 0.0

    X -= avg[:, None]
    X_T = X.t()
    c = dot(X, X_T)
    c *= 1. / fact
    return c.squeeze()

Now in numpy, this would transparently work with complex numbers, but I cannot simply feed a 3-d array with the last dimension being (real, imag) and hope it will work.

How can I adapt the calculation to obtain the complex covariance matrix with real and imaginary channels?

Output:
[For PyTorch implementation of cov() for complex matrices, skip the explanations and go to the last snippet]

Description of cov() operation

Let M be a HxW matrix, where each of the H rows corresponds to a variable of W complex observations.

Now let cov(M) be the HxH covariance matrix of the H variables of M (definition of numpy.cov()). It can be computed as follows (ignoring edge cases):

cov(M) = 1 / (W - 1) . M * M.T

with * matrix multiplication operator, and M.T tranpose of M.

Note: to clarify the next equations, let cov_prod(X, Y) = 1 / (W - 1) . X * Y.T, with X, Y HxW matrices. We thus have cov(M) = cov_prod(M, M).

So far, nothing new, this corresponds to the code you wrote (minus the y weighting and data checks for edge cases). Let's double-check that the Pytorch implementation of this formula corresponds to the Numpy one, for real-valued data:

import torch
import numpy as np

def cov(m, y=None):
    if y is not None:
        m = torch.cat((m, y), dim=0)
    m_exp = torch.mean(m, dim=1)
    x = m - m_exp[:, None]
    cov = 1 / (x.size(1) - 1) * x.mm(x.t())
    return cov

# Real-valued matrix:
M_np = np.random.rand(3, 2)
# Same matrix as torch.Tensor:
M = torch.from_numpy(M_np)

cov_real_np = np.cov(M_np)
cov_real = cov(M)
eq = np.allclose(cov_real_np, cov_real.numpy())
print("Numpy & Torch real covariance results equal? > {}".format(eq))
# Numpy & PyTorch real covariance results equal? > True

Extension to complex matrices

Now, how does this works for complex matrices?
From here, let M be of complex values, i.e. composed of H row-variables of W complex observations.
Furthermore, let A and B be the real-valued matrices such that M = A + i.B.

I will not go into the mathematical demonstration, which you can find here thanks to @zimzam, but in that case cov(M) can be decomposed as:

cov(M) = [cov_prod(A, A) + cov_prod(B, B)] + i.[-cov_prod(A, B) + cov_prod(B, A)]

This makes it straightforward to compute separately the real and imaginary components of cov(M), given the real and imaginary components of M (A and B).

Implementation & Validation

Find below an optimized implementation:

import torch
import numpy as np

def cov_complex(m_comp):
    # (adding further parameters such as `y` is left for exercise)
    # Supposing real and img are stored separately in the last dim:
    real, img = m_comp[..., 0], m_comp[..., 1]
    x_real = real - torch.mean(real, dim=1)[:, None]
    x_img = img - torch.mean(img, dim=1)[:, None]
    x_real_T = x_real.t()
    x_img_T = x_img.t()
    frac = 1 / (x_real.size(1) - 1)

    cov_real = frac * (x_real.mm(x_real_T) + x_img.mm(x_img_T))
    cov_img = frac * (-x_real.mm(x_img_T) + x_img.mm(x_real_T))
    return torch.stack((cov_real, cov_img), dim=-1)

# Matrix with real/img values stored separately in last dimension:
M_np = np.random.rand(3, 2, 2)
# Same matrix converted to np.complex format:
M_comp_np = M_np.view(dtype=np.complex128)[...,0]
# Same matrix as torch.Tensor:
M = torch.from_numpy(M_np)

cov_com_np = np.cov(M_comp_np)
cov_com = cov_complex(M)
eq = np.allclose(cov_com_np, cov_com.numpy().view(dtype=np.complex128)[...,0])
print("Numpy & Torch complex covariance results equal? > {}".format(eq))
# Numpy & PyTorch complex covariance results equal? > True