Instruction: Getting rid of maxpooling layer causes running cuda out memory error pytorch

Input:
Video card: gtx1070ti 8Gb, batchsize 64, input image size 128*128.
I had such UNET with resnet152 as encoder wich worket pretty fine:

class UNetResNet(nn.Module):

def __init__(self, encoder_depth, num_classes, num_filters=32, dropout_2d=0.2,
             pretrained=False, is_deconv=False):
    super().__init__()
    self.num_classes = num_classes
    self.dropout_2d = dropout_2d

    if encoder_depth == 34:
        self.encoder = torchvision.models.resnet34(pretrained=pretrained)
        bottom_channel_nr = 512
    elif encoder_depth == 101:
        self.encoder = torchvision.models.resnet101(pretrained=pretrained)
        bottom_channel_nr = 2048
    elif encoder_depth == 152:
        self.encoder = torchvision.models.resnet152(pretrained=pretrained)
        bottom_channel_nr = 2048

    else:
        raise NotImplementedError('only 34, 101, 152 version of Resnet are implemented')

    self.pool = nn.MaxPool2d(2, 2)

    self.relu = nn.ReLU(inplace=True)

    self.conv1 = nn.Sequential(self.encoder.conv1,
                               self.encoder.bn1,
                               self.encoder.relu,
                               self.pool) #from that pool layer I would like to get rid off

    self.conv2 = self.encoder.layer1
    self.conv3 = self.encoder.layer2
    self.conv4 = self.encoder.layer3
    self.conv5 = self.encoder.layer4
    self.center = DecoderCenter(bottom_channel_nr, num_filters * 8 *2, num_filters * 8, False)

    self.dec5 =  DecoderBlockV(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8,   is_deconv)
    self.dec4 = DecoderBlockV(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
    self.dec3 = DecoderBlockV(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2, is_deconv)
    self.dec2 = DecoderBlockV(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,
                               is_deconv)
    self.dec1 = DecoderBlockV(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
    self.dec0 = ConvRelu(num_filters, num_filters)
    self.final = nn.Conv2d(num_filters, num_classes, kernel_size=1)

def forward(self, x):
    conv1 = self.conv1(x)
    conv2 = self.conv2(conv1)
    conv3 = self.conv3(conv2)
    conv4 = self.conv4(conv3)
    conv5 = self.conv5(conv4)
    center = self.center(conv5)
    dec5 = self.dec5(torch.cat([center, conv5], 1))
    dec4 = self.dec4(torch.cat([dec5, conv4], 1))
    dec3 = self.dec3(torch.cat([dec4, conv3], 1))
    dec2 = self.dec2(torch.cat([dec3, conv2], 1))
    dec1 = self.dec1(dec2)
    dec0 = self.dec0(dec1)

    return self.final(F.dropout2d(dec0, p=self.dropout_2d))
# blocks
    class DecoderBlockV(nn.Module):
        def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
            super(DecoderBlockV2, self).__init__()
            self.in_channels = in_channels

            if is_deconv:
                self.block = nn.Sequential(
                    ConvRelu(in_channels, middle_channels),
                    nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
                                       padding=1),
                    nn.BatchNorm2d(out_channels),
                    nn.ReLU(inplace=True)

                )
            else:
                self.block = nn.Sequential(
                    nn.Upsample(scale_factor=2, mode='bilinear'),
                    ConvRelu(in_channels, middle_channels),
                    ConvRelu(middle_channels, out_channels),
                )

        def forward(self, x):
            return self.block(x)

class DecoderCenter(nn.Module):
    def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
        super(DecoderCenter, self).__init__()
        self.in_channels = in_channels

        if is_deconv:
            """
                Paramaters for Deconvolution were chosen to avoid artifacts, following
                link https://distill.pub/2016/deconv-checkerboard/
            """

            self.block = nn.Sequential(
                ConvRelu(in_channels, middle_channels),
                nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
                                   padding=1),
        nn.BatchNorm2d(out_channels),
                nn.ReLU(inplace=True)
            )
        else:
            self.block = nn.Sequential(
                ConvRelu(in_channels, middle_channels),
                ConvRelu(middle_channels, out_channels)

            )

    def forward(self, x):
        return self.block(x)

Then I edited my class looks to make it work without pooling layer:

class UNetResNet(nn.Module):
    def __init__(self, encoder_depth, num_classes, num_filters=32, dropout_2d=0.2,
                 pretrained=False, is_deconv=False):
        super().__init__()
        self.num_classes = num_classes
        self.dropout_2d = dropout_2d

        if encoder_depth == 34:
            self.encoder = torchvision.models.resnet34(pretrained=pretrained)
            bottom_channel_nr = 512
        elif encoder_depth == 101:
            self.encoder = torchvision.models.resnet101(pretrained=pretrained)
            bottom_channel_nr = 2048
        elif encoder_depth == 152:
            self.encoder = torchvision.models.resnet152(pretrained=pretrained)
            bottom_channel_nr = 2048
        else:
            raise NotImplementedError('only 34, 101, 152 version of Resnet are implemented')

        self.relu = nn.ReLU(inplace=True)

        self.input_adjust = nn.Sequential(self.encoder.conv1,
                                          self.encoder.bn1,
                                          self.encoder.relu)

        self.conv1 = self.encoder.layer1
        self.conv2 = self.encoder.layer2
        self.conv3 = self.encoder.layer3
        self.conv4 = self.encoder.layer4

        self.dec4 = DecoderBlockV(bottom_channel_nr, num_filters * 8 * 2, num_filters * 8, is_deconv)
        self.dec3 = DecoderBlockV(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8,    is_deconv)
        self.dec2 = DecoderBlockV(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2,    is_deconv)
        self.dec1 = DecoderBlockV(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,is_deconv)
        self.final = nn.Conv2d(num_filters * 2 * 2, num_classes, kernel_size=1)

    def forward(self, x):
        input_adjust = self.input_adjust(x)
        conv1 = self.conv1(input_adjust)
        conv2 = self.conv2(conv1)
        conv3 = self.conv3(conv2)
        center = self.conv4(conv3)
        dec4 = self.dec4(center) #now without centblock
        dec3 = self.dec3(torch.cat([dec4, conv3], 1))
        dec2 = self.dec2(torch.cat([dec3, conv2], 1))
        dec1 = F.dropout2d(self.dec1(torch.cat([dec2, conv1], 1)), p=self.dropout_2d)
        return self.final(dec1)

is_deconv - in both cases True. After changing it stop to work with batchsize 64, only with with size of 16 or with batchsize 64 but with resnet16 only - otherwise out of cuda memory. What am I doing wrong?

Full stack of error:

~/Desktop/ml/salt/open-solution-salt-identification-master/common_blocks/unet_models.py in forward(self, x)
    418         conv1 = self.conv1(input_adjust)
    419         conv2 = self.conv2(conv1)
--> 420         conv3 = self.conv3(conv2)
    421         center = self.conv4(conv3)
    422         dec4 = self.dec4(center)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     65     def forward(self, input):
     66         for module in self._modules.values():
---> 67             input = module(input)
     68         return input
     69

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torchvision-0.2.0-py3.6.egg/torchvision/models/resnet.py in forward(self, x)
     79
     80         out = self.conv2(out)
---> 81         out = self.bn2(out)
     82         out = self.relu(out)

Output:
The problem is that you do not have enough memory, as already mentioned in the comments.

To be more specific, the problem lies in the increased size due to the removal of the max pooling, as you already correctly narrowed it down. The point of max pooling - aside from the increased invariance of the setting - is to reduce the image size, such that it costs you less memory because you need to store less activations for the backpropagation part.

A good answer of what max pooling does might be helpful. My go-to one is the one on Quora.
As you also correctly figured out already is the fact that the batch size also plays a huge role in terms of memory consumption. Generally, it is also preferred to use smaller batch sizes anyways, as long as you don't have a skyrocketing processing time after.


Instruction: Why would Pytorch (CUDA) be running slow on GPU

Input:
I have been playing around with Pytorch on Linux for some time now and recently decided to try get more scripts to run with my GPU on my Windows desktop. Since trying this I have noticed a massive performance difference between my GPU execution time and my CPU execution time, on the same scripts, such that my GPU is significantly slow than CPU. To illustrate this I just a tutorial program found here (https://pytorch.org/tutorials/beginner/pytorch_with_examples.html#pytorch-tensors)

import torch
import datetime
print(torch.__version__)

dtype = torch.double
#device = torch.device("cpu")
device = torch.device("cuda:0")

# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10

# Create random input and output data
x = torch.randn(N, D_in, device=device, dtype=dtype)
y = torch.randn(N, D_out, device=device, dtype=dtype)

# Randomly initialize weights
w1 = torch.randn(D_in, H, device=device, dtype=dtype)
w2 = torch.randn(H, D_out, device=device, dtype=dtype)

start = datetime.datetime.now()
learning_rate = 1e-6
for t in range(5000):
    # Forward pass: compute predicted y
    h = x.mm(w1)
    h_relu = h.clamp(min=0)
    y_pred = h_relu.mm(w2)

    # Compute and print loss
    loss = (y_pred - y).pow(2).sum().item()
    #print(t, loss)

    # Backprop to compute gradients of w1 and w2 with respect to loss
    grad_y_pred = 2.0 * (y_pred - y)
    grad_w2 = h_relu.t().mm(grad_y_pred)
    grad_h_relu = grad_y_pred.mm(w2.t())
    grad_h = grad_h_relu.clone()
    grad_h[h < 0] = 0
    grad_w1 = x.t().mm(grad_h)

    # Update weights using gradient descent
    w1 -= learning_rate * grad_w1
    w2 -= learning_rate * grad_w2

end = datetime.datetime.now()

print(end-start)

I increased the number of Epoch's from 500 to 5000 as I have read that the first CUDA call is very slow due to initialisation. However the performance issue still exists.

With device = torch.device("cpu") the final time printed out is normal around 3-4 seconds, well device = torch.device("cuda:0") executes in around 13-15 seconds

I have reinstalled Pytorch a number of different ways (uninstalling the previous installation of course) and the problem still persists. I am hoping that someone can help me, if I have perhaps missed a set (didn't install some other API/program) or am doing something wrong in the code.

Python: v3.6

Pytorch:v0.4.1

GPU: NVIDIA GeForce GTX 1060 6GB

Any help would be appreciated :slight_smile:

Output:
Running on gpu could be expensive when you run with smaller batch size. If you put more data to gpu, means increasing the batch size, then you could observe significance amount of increase in data. Yes gpu is running better with float32 than double.
Try this

**

N, D_in, H, D_out = 128, 1000, 500, 10
dtype = torch.float32

**


Instruction: pyTorch can backward twice without setting retain_graph=True

Input:
As indicated in pyTorch tutorial,

  if you even want to do the backward on some part of the graph twice,
  you need to pass in retain_graph = True during the first pass.

However, I found the following codes snippet actually worked without doing so. I'm using pyTorch-0.4

x = torch.ones(2, 2, requires_grad=True)
y = x + 2
y.backward(torch.ones(2, 2)) # Note I do not set retain_graph=True
y.backward(torch.ones(2, 2)) # But it can still work!
print x.grad

output:

tensor([[ 2.,  2.],
        [ 2.,  2.]])

Could anyone explain? Thanks in advance!

Output:
The reason why it works w/o retain_graph=True in your case is you have very simple graph that probably would have no internal intermediate buffers, in turn no buffers will be freed, so no need to use retain_graph=True.

But everything is changing when adding one more extra computation to your graph:

Code:

x = torch.ones(2, 2, requires_grad=True)
v = x.pow(3)
y = v + 2

y.backward(torch.ones(2, 2))

print('Backward 1st time w/o retain')
print('x.grad:', x.grad)

print('Backward 2nd time w/o retain')

try:
    y.backward(torch.ones(2, 2))
except RuntimeError as err:
    print(err)

print('x.grad:', x.grad)

Output:

Backward 1st time w/o retain
x.grad: tensor([[3., 3.],
                [3., 3.]])
Backward 2nd time w/o retain
Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.
x.grad: tensor([[3., 3.],
                [3., 3.]]).

In this case additional internal v.grad will be computed, but torch doesn't store intermediate values (intermediate gradients etc), and with retain_graph=False v.grad will be freed after first backward.

So, if you want to backprop second time you need to specify retain_graph=True to "keep" the graph.

Code:

x = torch.ones(2, 2, requires_grad=True)
v = x.pow(3)
y = v + 2

y.backward(torch.ones(2, 2), retain_graph=True)

print('Backward 1st time w/ retain')
print('x.grad:', x.grad)

print('Backward 2nd time w/ retain')

try:
    y.backward(torch.ones(2, 2))
except RuntimeError as err:
    print(err)
print('x.grad:', x.grad)

Output:

Backward 1st time w/ retain
x.grad: tensor([[3., 3.],
                [3., 3.]])
Backward 2nd time w/ retain
x.grad: tensor([[6., 6.],
                [6., 6.]])


Instruction: How do I visualize a net in Pytorch?

Input:
import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as data
import torchvision.models as models
import torchvision.datasets as dset
import torchvision.transforms as transforms
from torch.autograd import Variable
from torchvision.models.vgg import model_urls
from torchviz import make_dot

batch_size = 3
learning_rate =0.0002
epoch = 50

resnet = models.resnet50(pretrained=True)
print resnet
make_dot(resnet)

I want to visualize resnet from the pytorch models. How can I do it? I tried to use torchviz but it gives an error:

'ResNet' object has no attribute 'grad_fn'

Output:
The make_dot expects a variable (i.e., tensor with grad_fn), not the model itself.
try:
x = torch.zeros(1, 3, 224, 224, dtype=torch.float, requires_grad=False)
out = resnet(x)
make_dot(out)  # plot graph of variable, not of a nn.Module


Instruction: Split dataset based on file names in pytorch Dataset

Input:
Is there a way to divide the dataset into training and testing based on the filenames. I have a folder containing two folders: input and output. Input folder has the images and output are the labels for that image. The file names in the input folder are something like input01_train.png and input01_test.png like shown below.

                          Dataset
                          /     \
                     Input       Output
                      |             |
           input01_train.png   output01_train.png
                    .                 .
                    .                 .
           input01_test.png    output01_test.png

The code I have only divides the dataset into inputs and labels not test and train.

class CancerDataset(Dataset):
  def __init__(self, dataset_folder):#,label_folder):
    self.dataset_folder = torchvision.datasets.ImageFolder(dataset_folder ,transform = transforms.Compose([transforms.Resize(512),transforms.ToTensor()]))
    self.label_folder = torchvision.datasets.ImageFolder(dataset_folder ,transform = transforms.Compose([transforms.Resize(512),transforms.ToTensor()]))

  def __getitem__(self,index):
    img = self.dataset_folder[index]
    label = self.label_folder[index]
    return img,label

  def __len__(self):
    return len(self.dataset_folder)

trainset = CancerDataset(dataset_folder = '/content/drive/My Drive/cancer_data/')
trainsetloader = DataLoader(trainset,batch_size = 1, shuffle = True,num_workers = 0,pin_memory = True)

I would like to be able to divide the train and test set by their names if that is possible .

Output:
You could load the images yourself in __getitem__, selecting only those that contain '_train.png' or '_test.png'.

class CancerDataset(Dataset):
    def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.Resize(512),transforms.ToTensor()]):
        self.datafolder = datafolder
        self.image_files_list = [s for s in os.listdir(datafolder) if
                                 '_%s.png' % datatype in s]
        # Same for the labels files
        self.label_files_list = ...
        self.transform = transform

    def __len__(self):
        return len(self.image_files_list)

    def __getitem__(self, idx):
        img_name = os.path.join(self.datafolder,
                                self.image_files_list[idx])
        image = Image.open(img_name)
        image = self.transform(image)
        # Same for the labels files
        label = .... # Load in etc
        label = self.transform(label)
        return image, label

Now you could make two datasets (trainset and testset).

trainset = CancerDataset(dataset_folder = '/content/drive/My Drive/cancer_data/', datatype='train')
testset = CancerDataset(dataset_folder = '/content/drive/My Drive/cancer_data/', datatype='test')


Instruction: How can I specify the flatten layer input size after many conv layers in PyTorch?

Input:
Here is my problem, I do a small test on CIFAR10 dataset, how can I specify the flatten layer input size in PyTorch? like the following, the input size is 16*5*5, however I don't know how to calculate this and I want to get the input size through some function.Can someone just write a simple function in this Net class and solve this?

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3,6,5)
        self.conv2 = nn.Conv2d(6,16,5)

        # HERE , the input size is 16*5*5, but I don't know how to get it.
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120,84)
        self.fc3 = nn.Linear(84,10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)),(2,2))
        x = F.max_pool2d(F.relu(self.conv2(x)),2)
        x = x.view(x.size()[0],-1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

Output:
There is no Flatten Layer in the Pytorch default. You can create a class like below. Cheers

class Flatten(nn.Module):
    def forward(self, input):
        return input.view(input.size(0), -1)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.flatten   = Flatten()  ## describing the layer
        self.conv1 = nn.Conv2d(3,6,5)
        self.conv2 = nn.Conv2d(6,16,5)

        # HERE , the input size is 16*5*5, but I don't know how to get it.
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120,84)
        self.fc3 = nn.Linear(84,10)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)),(2,2))
        x = F.max_pool2d(F.relu(self.conv2(x)),2)
        #x = x.view(x.size()[0],-1)
        x = self.flatten(x)   ### using of flatten layer
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x


Instruction: The similar function of "tf.Session().partial_run()" in Pytorch

Input:
I am trying to use Pytorch to reimplement a project, but I have some troubles to find a similar function of "partial_run" of Tensorflow. Can you show me what it is?
Thanks.

Output:
You don't need the partial run function, as Pytorch objects hold actual values (and are not just abstracted computation/input nodes in a graph that only hold values once the computation graph is triggered - with Session.(partial)run).

So when you code something in pytorch, you can (mostly) just think of it as writing numpy code. There, too, the objects you create when writing your code will hold actual values that you can manipulate.

Besides being more "traditional" this has the great consequence that Pytorch thus allows you to debug code using a traditional python debugger.


Instruction: Data preprocessing for custom dataset in pytorch (transform.Normalize)

Input:
I am new to Pytorch and CNN. I am kind of confused about Data Preprocessing. Not sure how to go about transform.Normalising the dataset (in essence how do you calculate mean and std v for your custom dataset ?)

I am loading my data using ImageFolder. The images are of different sizes.

train_transforms = transforms.Compose([transforms.Resize(size=224),
                                       transforms.ToTensor(),  transforms.Normalize((?), (?))
                                       ])
train_dataset = datasets.ImageFolder(root='roota/',
                                     transform=train_transforms)

Output:
If you're planning to train your network from scratch, you can calculate your dataset's statistics. The statistics of the dataset are calculated beforehand. You can use the ImageFolder to loop through the images to calculate the dataset statistics. For example, pseudo code -

for inputs, labels in dataloaders:
    # Calculate mean and std dev
    # save for later processing

Typically, CNNs are pretrained with other larger datasets, such as Imagenet, primarily to reduce the training time. If you are using a pretrained network, you can use the mean and std dev of the original dataset for your training.


Instruction: How to use CUDA stream in Pytorch?

Input:
I wanna use CUDA stream in Pytorch to parallel some computations, but I don't know how to do it.
For instance, if there's 2 tasks, A and B, need to be parallelized, I wanna do the following things:

stream0 = torch.get_stream()
stream1 = torch.get_stream()
with torch.now_stream(stream0):
    // task A
with torch.now_stream(stream1):
    // task B
torch.synchronize()
// get A and B's answer

How can I achieve the goal in real python code?

Output:
s1 = torch.cuda.Stream()
s2 = torch.cuda.Stream()
# Initialise cuda tensors here. E.g.:
A = torch.rand(1000, 1000, device = 'cuda')
B = torch.rand(1000, 1000, device = 'cuda')
# Wait for the above tensors to initialise.
torch.cuda.synchronize()
with torch.cuda.stream(s1):
    C = torch.mm(A, A)
with torch.cuda.stream(s2):
    D = torch.mm(B, B)
# Wait for C and D to be computed.
torch.cuda.synchronize()
# Do stuff with C and D.


Instruction: RuntimeError: expected stride to be a single integer value

Input:
I am new at Pytorch sorry for the basic question. The model gives me dimension mismatch error how to solve this ?
Maybe more than one problems in it.
Any help would be appriciated.
Thanks

class PR(nn.Module):
        def __init__(self):
            super(PR, self).__init__()
            self.conv1     = nn.Conv2d(3,6,kernel_size=5)
            self.conv2     = nn.Conv2d(6,1,kernel_size=2)
            self.dens1     = nn.Linear(300, 256)
            self.dens2     = nn.Linear(256, 256)
            self.dens3     = nn.Linear(512, 24)
            self.drop      = nn.Dropout()

        def forward(self, x):
            out = self.conv1(x)
            out = self.conv2(x)
            out = self.dens1(x)
            out = self.dens2(x)
            out = self.dens3(x)
            return out

model = PR()
input = torch.rand(28,28,3)
output = model(input)

Output:
Please have a look at the corrected code. I numbered the lines where I did corrections and described them below.

class PR(torch.nn.Module):
    def __init__(self):
        super(PR, self).__init__()
        self.conv1     = torch.nn.Conv2d(3,6, kernel_size=5) # (2a) in 3x28x28 out 6x24x24
        self.conv2     = torch.nn.Conv2d(6,1, kernel_size=2) # (2b) in 6x24x24 out 1x23x23 (6)
        self.dens1     = torch.nn.Linear(529, 256) # (3a)
        self.dens2     = torch.nn.Linear(256, 256)
        self.dens3     = torch.nn.Linear(256, 24) # (4)
        self.drop      = torch.nn.Dropout()

    def forward(self, x):
        out = self.conv1(x)
        out = self.conv2(out) # (5)

        out = out.view(-1, 529) # (3b)

        out = self.dens1(out)
        out = self.dens2(out)
        out = self.dens3(out)
        return out

model = PR()
ins = torch.rand(1, 3, 28, 28) # (1)
output = model(ins)

First of all, pytorch handles image tensors (you perform 2d convolution therefore I assume this is an image input) as follows: [batch_size x image_depth x height width]
It is important to understand how the convolution with kernel, padding and stride works. In your case kernel_size is 5 and you have no padding (and stride 1). This means that the dimensions of the feature-map gets reduced (as depicted). In your case the first conv. layer takes a 3x28x28 tensor and produces a 6x24x24 tensor, the second one takes 6x24x24 out 1x23x23. I find it very useful to have comments with the in and out tensor dimensions next to the definition conv layers (see in the code above)

Here you need to "flatten" the [batch_size x depth x height x width] tensor to [batch_size x fully connected input]. This can be done via tensor.view().
There was a wrong input for the linear layer
Each operation in the forward-pass took the input value x, instead I think you might want to pass the results of each layer to the next one

Altough this code is now runnable, it does not mean that it makes perfect sense. The most important thing (for neural networks in general i would say) are activation functions. These are missing completely.

For getting started with neural networks in pytorch I can highly recommend the great pytorch tutorials: https://pytorch.org/tutorials/ (I would start with the 60min blitz tutorial)

Hope this helps!


Instruction: Pytorch not building with cmake in Developer Console

Input:
I've mostly used languages with simple IDEs until now, so I don't have the best knowledge of compiling and running git and cmake and everything else through command line. I need to use Pytorch for a project though, so it's necessary to use those skills. I'm installing it according to the tutorial for windows found here:

https://caffe2.ai/docs/getting-started.html?platform=windows&configuration=compile

I've gotten to the point where I'm running build_windows.bat, but I'm getting this output with an error from the Developer Command Prompt.

The system cannot find the drive specified.
Requirement already satisfied: pyyaml in g:\programs\python27\lib\site-packages (3.13)
CAFFE2_ROOT=G:\Programs\Caffe2\pytorch\scripts\..
CMAKE_GENERATOR="Visual Studio 14 2015 Win64"
CMAKE_BUILD_TYPE=Release
-- Selecting Windows SDK version  to target Windows 10.0.17134.
CMake Error at CMakeLists.txt:6 (project):
  Failed to run MSBuild command:

    MSBuild.exe

  to get the value of VCTargetsPath:

    Microsoft (R) Build Engine version 15.8.169+g1ccb72aefa for .NET Framework
    Copyright (C) Microsoft Corporation. All rights reserved.

    Build started 9/25/2018 4:20:32 PM.
    Project "G:\Programs\Caffe2\pytorch\build\CMakeFiles\3.12.2\VCTargetsPath.vcxproj" on node 1 (default targets).
    G:\Programs\Caffe2\pytorch\build\CMakeFiles\3.12.2\VCTargetsPath.vcxproj(14,2): error MSB4019: The imported project "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\VC\VCTargets\Microsoft.Cpp.Default.props" was not found. Confirm that the path in the <Import> declaration is correct, and that the file exists on disk.
    Done Building Project "G:\Programs\Caffe2\pytorch\build\CMakeFiles\3.12.2\VCTargetsPath.vcxproj" (default targets) -- FAILED.

    Build FAILED.

    "G:\Programs\Caffe2\pytorch\build\CMakeFiles\3.12.2\VCTargetsPath.vcxproj" (default target) (1) ->
      G:\Programs\Caffe2\pytorch\build\CMakeFiles\3.12.2\VCTargetsPath.vcxproj(14,2): error MSB4019: The imported project "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\VC\VCTargets\Microsoft.Cpp.Default.props" was not found. Confirm that the path in the <Import> declaration is correct, and that the file exists on disk.

        0 Warning(s)
        1 Error(s)

    Time Elapsed 00:00:00.15

  Exit code: 1

-- Configuring incomplete, errors occurred!
See also "G:/Programs/Caffe2/pytorch/build/CMakeFiles/CMakeOutput.log".
"Caffe2 building failed"

The CMakeOutput.log file only contains this:

The system is: Windows - 10.0.17134 - AMD64

Which isn't very useful. I'm not sure what I'm doing wrong here.

Output:
If you just need to test or experiment with pytorch I suggest that you first try to install it through the pip package. It is much easier.

If you really need to install it from source, then I suggest that you read the build_windows.bat file to check that it really suits your configuration and modify it if needed. Make sure you are targeting the correct Visual Studio version for example.

The error you are getting doesn't seem to come from the pytorch project itself.


Instruction: PyTorch softmax with dim

Input:
Which dimension should softmax be applied to ?

This code :

%reset -f

import torch.nn as nn
import numpy as np
import torch

my_softmax = nn.Softmax(dim=-1)

mu, sigma = 0, 0.1 # mean and standard deviation

train_dataset = []
image = []
image_x = np.random.normal(mu, sigma, 24).reshape((3 , 4, 2))
train_dataset.append(image_x)

x = torch.tensor(train_dataset).float()

print(x)
print(my_softmax(x))
my_softmax = nn.Softmax(dim=1)
print(my_softmax(x))

prints following :

tensor([[[[-0.1500,  0.0243],
          [ 0.0226,  0.0772],
          [-0.0180, -0.0278],
          [ 0.0782, -0.0853]],

         [[-0.0134, -0.1139],
          [ 0.0385, -0.1367],
          [-0.0447,  0.1493],
          [-0.0633, -0.2964]],

         [[ 0.0123,  0.0061],
          [ 0.1086, -0.0049],
          [-0.0918, -0.1308],
          [-0.0100,  0.1730]]]])
tensor([[[[ 0.4565,  0.5435],
          [ 0.4864,  0.5136],
          [ 0.5025,  0.4975],
          [ 0.5408,  0.4592]],

         [[ 0.5251,  0.4749],
          [ 0.5437,  0.4563],
          [ 0.4517,  0.5483],
          [ 0.5580,  0.4420]],

         [[ 0.5016,  0.4984],
          [ 0.5284,  0.4716],
          [ 0.5098,  0.4902],
          [ 0.4544,  0.5456]]]])
tensor([[[[ 0.3010,  0.3505],
          [ 0.3220,  0.3665],
          [ 0.3445,  0.3230],
          [ 0.3592,  0.3221]],

         [[ 0.3450,  0.3053],
          [ 0.3271,  0.2959],
          [ 0.3355,  0.3856],
          [ 0.3118,  0.2608]],

         [[ 0.3540,  0.3442],
          [ 0.3509,  0.3376],
          [ 0.3200,  0.2914],
          [ 0.3289,  0.4171]]]])

So first tensor is prior to softmax being applied, second tensor is result of softmax applied to tensor with dim=-1 and third tensor is result of softmax applied to tensor with dim=1 .

For result of first softmax can see corresponding elements sum to 1, for example [ 0.4565,  0.5435] -> 0.4565 + 0.5435 == 1.

What is summing to 1 as result of of second softmax ?

Which dim value should I choose ?

Update : The dimension (3 , 4, 2) corresponds to image dimension where 3 is the RGB value , 4 is the number of horizontal pixels (width) , 2 is the number of vertical pixels (height). This is an image classification problem. I'm using cross entropy loss function. Also, I'm using softmax in final layer in order to back-propagate probabilities.

Output:
You have a 1x3x4x2 tensor train_dataset. Your softmax function's dim parameter determines across which dimension to perform Softmax operation. First dimension is your batch dimension, second is depth, third is rows and last one is columns. Please look at picture below (sorry for horrible drawing) to understand how softmax is performed when you specify dim as 1.

In short, sum of each corresponding entry of your 4x2 matrices are equal to 1.

Update: The question which dimension the softmax should be applied depends on what data your tensor store, and what is your goal.

Update: For image classification task, please see the tutorial on official pytorch website. It covers basics of image classification with pytorch on a real dataset and its a very short tutorial. Although that tutorial does not perform Softmax operation, what you need to do is just use torch.nn.functional.log_softmax on output of last fully connected layer. See MNIST classifier with pytorch for a complete example. It does not matter whether your image is RGB or grayscale after flattening it for fully connected layers (also keep in mind that same code for MNIST example might not work for you, depends on which pytorch version you use).


Instruction: tensor - box plot from a tensor

Input:
I'm trying to create a box between two variables probability (y axis) and flowername ( x- axis) .Probability is a tensor. For the flower name, I have to pick the name from a dictionary (flower_dict) where the key is referenced from another tensor, class Index. How do I create box plot ? ANy help is appreciate

print("Probability:", probs)
Probability: tensor([[ 0.9961,  0.0020,  0.0011,  0.0005,  0.0001]], device='cuda:0')

print("Class Index:", classes)
Class Index: tensor([[  21,  3,  45,  34,  27]], device='cuda:0')

print(flower_dict)
{'21': 'fire lily', '3': 'canterbury bells', '45': 'bolero deep blue', '1': 'pink primrose', '34': 'mexican aster', '27': 'prince of wales feathers', '7': 'moon orchid', '16': 'globe-flower', '25': 'grape hyacinth', '26': 'corn poppy', '79': 'toad lily', '39': 'siam tulip', '24': 'red ginger'}

Output:
I guess you mean creating a bar plot here (with box-plots you usually depict distributions like for examining features etc.; in the iris flower case for example you might want to examine sepal-length in a box plot). If bar plot is what you want, then you can try the following code:

import numpy as np
import matplotlib.pyplot as plt

y = probs.flatten().numpy()
x = [flower_dict[str(i)] for i in classes.flatten().numpy()]

p1 = plt.bar(x, y, width)

plt.ylabel('Probability')
plt.xticks(ind, x)

plt.show()

Hope this helps!


Instruction: How to compensate if I cant do a large batch size in neural network

Input:
I am trying to run an action recognition code from GitHub. The original code used a batch size of 128 with 4 GPUS. I only have two gpus so I cannot match their bacth size number. Is there anyway I can compensate this difference in batch. I saw somewhere that iter_size might compensate according to a formula effective_batchsize= batch_size*iter_size*n_gpu. what is iter_size in this formula?
I am using PYthorch not Caffe.

Output:
In pytorch, when you perform the backward step (calling loss.backward() or similar) the gradients are accumulated in-place. This means that if you call loss.backward() multiple times, the previously calculated gradients are not replaced, but in stead the new gradients get added on to the previous ones. That is why, when using pytorch, it is usually necessary to explicitly zero the gradients between minibatches (by calling optimiser.zero_grad() or similar).

If your batch size is limited, you can simulate a larger batch size by breaking a large batch up into smaller pieces, and only calling optimiser.step() to update the model parameters after all the pieces have been processed.

For example, suppose you are only able to do batches of size 64, but you wish to simulate a batch size of 128. If the original training loop looks like:

optimiser.zero_grad()
loss = model(batch_data) # batch_data is a batch of size 128
loss.backward()
optimiser.step()

then you could change this to:

optimiser.zero_grad()

smaller_batches = batch_data[:64], batch_data[64:128]
for batch in smaller_batches:
    loss = model(batch) / 2
    loss.backward()

optimiser.step()

and the updates to the model parameters would be the same in each case (apart maybe from some small numerical error). Note that you have to rescale the loss to make the update the same.


Instruction: PyTorch CUDA vs Numpy for arithmetic operations? Fastest?

Input:
I performed element-wise multiplication using Torch with GPU support and Numpy using the functions below and found that Numpy loops faster than Torch which shouldn't be the case, I doubt.

I want to know how to perform general arithmetic operations with Torch using GPU.

Note: I ran these code snippets in Google Colab notebook

Define the default tensor type to enable global GPU flag

torch.set_default_tensor_type(torch.cuda.FloatTensor if
                              torch.cuda.is_available() else
                              torch.FloatTensor)

Initialize Torch variables

x = torch.Tensor(200, 100)  # Is FloatTensor
y = torch.Tensor(200,100)

Function in question

def mul(d,f):
    g = torch.mul(d,f).cuda()  # I explicitly called cuda() which is not necessary
    return g

When call the function above as
%timeit mul(x,y)

Returns:

  The slowest run took 10.22 times longer than the fastest. This could
  mean hat   an intermediate result is being cached.  10000 loops, best
  of 3: 50.1 µs per loop

Now trial with numpy,

Used the same values from torch variables

x_ = x.data.cpu().numpy()
y_ = y.data.cpu().numpy()

def mul_(d,f):
    g = d*f
    return g

%timeit mul_(x_,y_)

Returns

  The slowest run took 12.10 times longer than the fastest. This could
  mean that an intermediate result is being cached.  100000 loops, best
  of 3: 7.73 µs per loop

Needs some help to understand GPU enabled Torch operations.

Output:
GPU operations have to additionally get memory to/from the GPU
The problem is that your GPU operation always has to put the input on the GPU memory, and
then retrieve the results from there, which is a quite costly operation.
NumPy, on the other hand, directly processes the data from the CPU/main memory, so there is almost no delay here. Additionally, your matrices are extremely small, so even in the best-case scenario, there should only be a minute difference.
This is also partially the reason why you use mini-batches when training on a GPU in neural networks: Instead of having several extremely small operations, you now have "one big bulk" of numbers that you can process in parallel.
Also note that GPU clock speeds are generally way lower than CPU clocks, so the GPU only really shines because it has way more cores. If your matrix does not utilize all of them fully, you are also likely to see a faster result on your CPU.
TL;DR: If your matrix is big enough, you will eventually see a speed-up in CUDA than Numpy, even with the additional cost of the GPU transfer.


Instruction: pytorch passing architecture type with argprse

Input:
Using Pytorch. When passing architecture type by using the following code:

parser.add_argument('-arch', action='store',
                    dest='arch',
                    default= str('vgg16'))

When using the name of the architecture with the following code:

model = models.__dict__['{!r}'.format(results.arch)](pretrained=True)

I get the following error:

  model = models.dict'{!r}'.format(results.arch)
  KeyError: "'vgg16'"

What am I doing wrong?

Output:
You got KeyError meaning your imported models do not include 'vgg16' as one of the known models.
Check what models you do have by printing

print(models.__dict__.keys())

This should allow you to know what models you import and which are missing, then you can look into your imports and see where 'vgg16' got lost.


Instruction: How to remove the last FC layer from a ResNet model in PyTorch?

Input:
I am using a ResNet152 model from PyTorch. I'd like to strip off the last FC layer from the model. Here's my code:

from torchvision import datasets, transforms, models
model = models.resnet152(pretrained=True)
print(model)

When I print the model, the last few lines look like this:

    (2):  Bottleneck(
      (conv1):  Conv2d(2048,  512,  kernel_size=(1,  1),  stride=(1,  1),  bias=False)
      (bn1):  BatchNorm2d(512,  eps=1e-05,  momentum=0.1,  affine=True,  track_running_stats=True)
      (conv2):  Conv2d(512,  512,  kernel_size=(3,  3),  stride=(1,  1),  padding=(1,  1),  bias=False)
      (bn2):  BatchNorm2d(512,  eps=1e-05,  momentum=0.1,  affine=True,  track_running_stats=True)
      (conv3):  Conv2d(512,  2048,  kernel_size=(1,  1),  stride=(1,  1),  bias=False)
      (bn3):  BatchNorm2d(2048,  eps=1e-05,  momentum=0.1,  affine=True,  track_running_stats=True)
      (relu):  ReLU(inplace)
    )
  )
  (avgpool):  AvgPool2d(kernel_size=7,  stride=1,  padding=0)
  (fc):  Linear(in_features=2048,  out_features=1000,  bias=True)
)

I want to remove that last fc layer from the model.

I found an answer here on SO (How to convert pretrained FC layers to CONV layers in Pytorch), where mexmex seems to provide the answer I'm looking for:

list(model.modules()) # to inspect the modules of your model
my_model = nn.Sequential(*list(model.modules())[:-1]) # strips off last linear layer

So I added those lines to my code like this:

model = models.resnet152(pretrained=True)
list(model.modules()) # to inspect the modules of your model
my_model = nn.Sequential(*list(model.modules())[:-1]) # strips off last linear layer
print(my_model)

But this code doesn't work as advertised -- as least not for me. The rest of this post is a detailed explanation of why that answer doesn't work so this question doesn't get closed as a duplicate.

First, the printed model is nearly 5x larger than before. I see the same model as before, but followed by what appears to be a repeat of the model, but perhaps flattened.

    (2): Bottleneck(
      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace)
    )
  )
  (avgpool): AvgPool2d(kernel_size=7, stride=1, padding=0)
  (fc): Linear(in_features=2048, out_features=1000, bias=True)
)
(1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
(2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(3): ReLU(inplace)
(4): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
(5): Sequential(
  . . . this goes on for ~1600 more lines . . .
  (415): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (416): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (417): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (418): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)
  (419): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (420): ReLU(inplace)
  (421): AvgPool2d(kernel_size=7, stride=1, padding=0)
)

Second, the fc layer is still there -- and the Conv2D layer after it looks just like the first layer of ResNet152.

Third, if I try to invoke my_model.forward(), pytorch complains about a size mismatch. It expects size [1, 3, 224, 224], but the input was [1, 1000]. So it looks like a copy of the entire model (minus the fc layer) is getting appended to the original model.

Bottom line, the only answer I found on SO doesn't actually work.

Output:
For ResNet model, you can use children attribute to access layers since ResNet model in pytorch consist of nn modules. (Tested on pytorch 0.4.1)

model = models.resnet152(pretrained=True)
newmodel = torch.nn.Sequential(*(list(model.children())[:-1]))
print(newmodel)

Update: Although there is not an universal answer for the question that can work on all pytorch models, it should work on all well structured ones. Existing layers you add to your model (such as torch.nn.Linear, torch.nn.Conv2d, torch.nn.BatchNorm2d...) all based on torch.nn.Module class. And if you implement a custom layer and add that to your network you should inherit it from pytorch's torch.nn.Module class. As written in documentation, children attribute lets you access the modules of your class/model/network.

def children(self):
        r"""Returns an iterator over immediate children modules.

Update: It is important to note that children() returns "immediate" modules, which means if last module of your network is a sequential, it will return whole sequential.


Instruction: Given 4 points, how to crop a quadrilateral from an image in pytorch/torchvision?

Input:
Problem is pretty straightforward -

I want to crop a quadrilateral from an image in pytorch/torchvision. Given, I have four coordinates of the corners of this quadrilateral.

Please note that these four points confine a quadrilateral within themselves which may or may not be a rectangle. So please refrain from suggesting answers involving slicing of the image.

Please comment if I have missed any relevant detail.

Output:
Quoting https://stackoverflow.com/a/30902423/4982729 I could extract the patch using opencv as -

import numpy as np
import cv2

pts = np.array([[542, 107], [562, 102], [582, 110], [598, 142], [600, 192], [601, 225], [592, 261], [572, 263], [551, 245], [526, 220], [520, 188], [518, 152], [525, 127], [524, 107]], dtype=np.int32)

mask = np.zeros((img.shape[0], img.shape[1]))

cv2.fillConvexPoly(mask, pts, 1)
mask = mask.astype(np.bool)

out = np.zeros_like(img)
out[mask] = img[mask]

and then I could convert the numpy array to torch's Variable manually. This seems to raise no errors even when I am forming torch's graph for neural networks.

    outputs = model(images)
    o = outputs.data.cpu().numpy()
    #
    # do your opencv stuff here to o
    #
    o = torch.Tensor(o).to(device)
    outputs = Variable(o, requires_grad=True)


Instruction: too many arguments in a Pytorch custom nn module forward function

Input:
I'm trying to make a neural network in pytorch that has a variable number of layers. My problem is that apparently I am passing some sort of iterable with more than one item to a linear layer which can only take one argument. I just can't see why.

So here is some code. First I created my own module and import it to my notebook later

import torch

class NNet(torch.nn.Module):
    def __init__(self, layer_shapes, activation_functions):
        super(NNet, self).__init__()
        assert len(layer_shapes) == len(activation_functions) + 1
        self.layer_shapes = layer_shapes
        self.activation_functions = activation_functions

        linear_functions = list()
        for i in range(len(self.layer_shapes)-1):
            linear_functions.append(torch.nn.Linear(
                    self.layer_shapes[i], self.layer_shapes[i+1]))

        self.linear_functions = linear_functions

    def parameters(self):
        parameters = list()
        for function in self.linear_functions:
            parameters = parameters+list(function.parameters())

        return parameters

    def forward(self, x):
        assert x.shape[1] == self.layer_shapes[0]
        y = x
        for i in range(len(self.layer_shapes)-1):
            lin = self.linear_functions[i](y)
            y = self.activation_functions[i](lin)
        return y

In the notebook, the error is in the forward function at y = self.activation_functions[i](self.linear_functions[i](y))

Now I try to use the MNIST dataset supplied by torchvision and use my own module.

batch_size = 100
epochs = 500
learning_rate = 0.001

train_set = torchvision.datasets.MNIST(root =
                                       '../../data',
                                        train=True,
                          transform=torchvision.transforms.ToTensor(),
                                        download=True)

test_set = torchvision.datasets.MNIST(root =
                                      '../../data',
                                      train=False,
                          transform=torchvision.transforms.ToTensor(),
                                   download=True)

train_loader = torch.utils.data.DataLoader(dataset=train_set,
                                           batch_size=batch_size,
                                           shuffle=True)

test_loader = torch.utils.data.DataLoader(dataset=test_set,
                                          batch_size=batch_size,
                                          shuffle=False)

model = nnet.NNet([784, 16, 10], [torch.nn.Tanh,
                                  torch.nn.Softmax(dim=1)])

loss_function = torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

loss_items = list()

for t in range(epochs):
    for i, (images, labels) in enumerate(train_loader):
        images = images.reshape(-1,28*28)

        outputs = model(images)
        loss = loss_function(outputs, labels)
        loss_items.append(loss.item())

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

This last for loop yields the error:

TypeError                      Traceback (most recent call last)
<ipython-input-6-4ccb4b105a41> in <module>()
      5         images = images.reshape(-1,28*28)
      6
----> 7         outputs = model(images)
      8         loss = loss_function(outputs, labels)
      9         loss_items.append(loss.item())

~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in
__call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/Desktop/Archive/Computing/Projects/Python/ai/neural_network.py in
forward(self, x)
     28         for i in range(len(self.layer_shapes)-1):
     29             lin = self.linear_functions[i](y)
---> 30             y = self.activation_functions[i](lin)
     31         return y
     32

TypeError: __init__() takes 1 positional argument but 2 were given

I'm sure someone could tell me why this is happening, but could someone please give me a helpful strategy for debugging here? I'm new to pytorch and I doubt this will be the last trouble that I have. So a strategy for investigating these things would be helpful.

I would appreciate any input.

Output:
In the definition of model I forgot the parentheses on the torch.nn.Tanh class.  It should be torch.nn.Tanh()

I keep thinking that these are functions and not classes. I still have some things to fix, but I'm glad I saw that. So frustrating. I found it by basically putting assert and print statements all over my code.


Instruction: Combining Neural Networks Pytorch

Input:
I have 2 images as input, x1 and x2  and try to use convolution as a similarity measure. The idea is that the learned weights substitute more traditional measure of similarity (cross correlation, NN, ...). Defining my forward function as follows:

def forward(self,x1,x2):
    out_conv1a = self.conv1(x1)
    out_conv2a = self.conv2(out_conv1a)
    out_conv3a = self.conv3(out_conv2a)

    out_conv1b = self.conv1(x2)
    out_conv2b = self.conv2(out_conv1b)
    out_conv3b = self.conv3(out_conv2b)

Now for the similarity measure:

out_cat = torch.cat([out_conv3a, out_conv3b],dim=1)
futher_conv = nn.Conv2d(out_cat)

My question is as follows:

1) Would Depthwise/Separable Convolutions as in the google paper yield any advantage over 2d convolution of the concatenated input. For that matter can convolution be a similarity measure, cross correlation and convolution are very similar.

2) It is my understanding that the groups=2 option in conv2d would provide 2 separate inputs to train weights with, in this case each of the previous networks weights. How are these combined afterwards?

For a basic concept see here.

Output:
Using a nn.Conv2d layer you assume weights are trainable parameters. However, if you want to filter one feature map with another, you can dive deeper and use torch.nn.functional.conv2d to explicitly define both input and filter yourself:

out = torch.nn.functional.conv2d(out_conv3a, out_conv3b)


Instruction: pytorch out of GPU memory

Input:
I am trying to implement Yolo-v2 in pytorch. However, I seem to be running out of memory just passing data through the network. The model is large and is shown below. However, I feel like I'm doing something stupid here with my network (like not freeing memory somewhere). The network works as expected on cpu.

The test code (where memory runs out) is:

x = torch.rand(32,3,416, 416).cuda()
model = Yolov2().cuda()
y = model(x.float())

Question

Anything that is obviously wrong with my model?
How do I make this more efficient with memory?
Other comments?

The model:

import torch
from torch import nn
import torch.nn.functional as F

class Yolov2(nn.Module):

    def __init__(self):
        super(Yolov2, self).__init__()

        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm1 = nn.BatchNorm2d(32)

        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm2 = nn.BatchNorm2d(64)

        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm3 = nn.BatchNorm2d(128)
        self.conv4 = nn.Conv2d(in_channels=128, out_channels=64, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm4 = nn.BatchNorm2d(64)
        self.conv5 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm5 = nn.BatchNorm2d(128)

        self.conv6 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm6 = nn.BatchNorm2d(256)
        self.conv7 = nn.Conv2d(in_channels=256, out_channels=128, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm7 = nn.BatchNorm2d(128)
        self.conv8 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm8 = nn.BatchNorm2d(256)

        self.conv9 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm9 = nn.BatchNorm2d(512)
        self.conv10 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm10 = nn.BatchNorm2d(256)
        self.conv11 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm11 = nn.BatchNorm2d(512)
        self.conv12 = nn.Conv2d(in_channels=512, out_channels=256, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm12 = nn.BatchNorm2d(256)
        self.conv13 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm13 = nn.BatchNorm2d(512)

        self.conv14 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm14 = nn.BatchNorm2d(1024)
        self.conv15 = nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm15 = nn.BatchNorm2d(512)
        self.conv16 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm16 = nn.BatchNorm2d(1024)
        self.conv17 = nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=1, stride=1, padding=0, bias=False)
        self.batchnorm17 = nn.BatchNorm2d(512)
        self.conv18 = nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm18 = nn.BatchNorm2d(1024)

        self.conv19 = nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm19 = nn.BatchNorm2d(1024)
        self.conv20 = nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm20 = nn.BatchNorm2d(1024)

        self.conv21 = nn.Conv2d(in_channels=3072, out_channels=1024, kernel_size=3, stride=1, padding=1, bias=False)
        self.batchnorm21 = nn.BatchNorm2d(1024)

        self.conv22 = nn.Conv2d(in_channels=1024, out_channels=125, kernel_size=1, stride=1, padding=0)

    def reorg_layer(self, x):
        stride = 2
        batch_size, channels, height, width = x.size()
        new_ht = int(height/stride)
        new_wd = int(width/stride)
        new_channels = channels * stride * stride

#         from IPython.core.debugger import Tracer; Tracer()()
        passthrough = x.permute(0, 2, 3, 1)
        passthrough = passthrough.contiguous().view(-1, new_ht, stride, new_wd, stride, channels)
        passthrough = passthrough.permute(0, 1, 3, 2, 4, 5)
        passthrough = passthrough.contiguous().view(-1, new_ht, new_wd, new_channels)
        passthrough = passthrough.permute(0, 3, 1, 2)
        return passthrough

    def forward(self, x):
        out = F.max_pool2d(F.leaky_relu(self.batchnorm1(self.conv1(x)), negative_slope=0.1), 2, stride=2)
        out = F.max_pool2d(F.leaky_relu(self.batchnorm2(self.conv2(out)), negative_slope=0.1), 2, stride=2)

        out = F.leaky_relu(self.batchnorm3(self.conv3(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm4(self.conv4(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm5(self.conv5(out)), negative_slope=0.1)
        out = F.max_pool2d(out, 2, stride=2)

        out = F.leaky_relu(self.batchnorm6(self.conv6(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm7(self.conv7(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm8(self.conv8(out)), negative_slope=0.1)
        out = F.max_pool2d(out, 2, stride=2)

        out = F.leaky_relu(self.batchnorm9(self.conv9(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm10(self.conv10(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm11(self.conv11(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm12(self.conv12(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm13(self.conv13(out)), negative_slope=0.1)
#         from IPython.core.debugger import Tracer; Tracer()()
        passthrough = self.reorg_layer(out)
        out = F.max_pool2d(out, 2, stride=2)

        out = F.leaky_relu(self.batchnorm14(self.conv14(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm15(self.conv15(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm16(self.conv16(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm17(self.conv17(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm18(self.conv18(out)), negative_slope=0.1)

        out = F.leaky_relu(self.batchnorm19(self.conv19(out)), negative_slope=0.1)
        out = F.leaky_relu(self.batchnorm20(self.conv20(out)), negative_slope=0.1)

        out = torch.cat([passthrough, out], 1)
        out = F.leaky_relu(self.batchnorm21(self.conv21(out)), negative_slope=0.1)
        out = self.conv22(out)

        return out

Additional Info:

Torch version is '0.4.1.post2'
Running on aws p2.xlarge (limit 12gb GPU memory).
The number of parameters for this model is 67137565. Which would occupy <500MB.
this thread from pytorch might be related.

Output:
I would try to use smaller batch sizes. Start from 1 and then check what is your maximum.
I can also try to reduce your input Tensor dimensions.
Your network is not so small for your GPU


Instruction: Is there any way I can download the pre-trained models available in PyTorch to a specific path?

Input: I am referring to the models that can be found here: https://pytorch.org/docs/stable/torchvision/models.html#torchvision-models

Output:
As, @dennlinger mentioned in his answer : torch.utils.model_zoo, is being internally called when you load a pre-trained model.

More specifically, the method: torch.utils.model_zoo.load_url() is being called every time a pre-trained model is loaded. The documentation for the same, mentions:

  The default value of model_dir is $TORCH_HOME/models where
  $TORCH_HOME defaults to ~/.torch.

  The default directory can be overridden with the $TORCH_HOME
  environment variable.

This can be done as follows:

import torch
import torchvision
import os

# Suppose you are trying to load pre-trained resnet model in directory- models\resnet

os.environ['TORCH_HOME'] = 'models\\resnet' #setting the environment variable
resnet = torchvision.models.resnet18(pretrained=True)

I came across the above solution by raising an issue in the PyTorch's GitHub repository:
https://github.com/pytorch/vision/issues/616

This led to an improvement in the documentation i.e. the solution mentioned above.


Instruction: Find all ReLU layer in a torchvision model

Input:
After I fetch a pre-trained model from torchvision.models, I want all the ReLU instance to register_backward_hook(f),which is like this:

for pos, module in self.model.features._modules.items():
    for sub_module in module:
        if isinstance(module, ReLU):
            module.register_backward_hook(f)

The problems for me is how to find all ReLU in a model. For densenet161, the ReLU exists not only in model.features._modules but also in self-defined dense layer, eg. model.features._modules['denseblock1'][0]. For  resnet151, the ReLU exists in model._modules and its self-define layer, eg model._modules['layer1'].

Is there any way to find all ReLU inside a model?

Output:
A more elegant way to iterate over all components of a model is using modules()
method:

from torch import nn

for module in self.model.modules():
  if isinstance(module, nn.ReLU):
    module.register_backward_hook(f)

If you do not want to get all sub-modules, only the immediate ones, you may consider using children() method instead of modules(). You can also get the name of the sub module using named_modules() method.


Instruction: Using TPUs with PyTorch

Input:
I am trying to use Google Cloud's TPU from Colab. I was able to do it following the tutorial by using Tensorflow.

Does anybody know if it is possible to make use of the TPUs using PyTorch?
If so how can I do it? Do you have any example?

Output:
Check out our repository pytorch/xla where you can start training PyTorch models on TPUs.

Also, you can even use free TPUs on Colab with PyTorch with these Colab notebooks.


Instruction: PyTorch - How to get learning rate during training?

Input:
While training, I'd like to know the value of learning_rate.
What should I do?

It's my code, like this:

my_optimizer = torch.optim.SGD(my_model.parameters(),
                               lr=0.001,
                               momentum=0.99,
                               weight_decay=2e-3)

Thank you.

Output:
For only one parameter group like in the example you've given, you can use this function and call it during training to get the current learning rate:

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']


Instruction: CNN weights at input image location

Input:
Given a CNN, say AlexNet:

How could one relate kernel locations at the 3rd conv block, i.e 13x13 filter size to the input image. I want to compare the filters at different locations.
I was thinking of just bilinearly upsampling the location, from 13x13 to 224x224,

nn.Upsample(size = (224,224), mode='bilinear')

however it's hard to justify local correspondence.

Output:
Using adaptive pooling, with adaptive pooling, one can reduce any feature map size.

Adaptive max pooling


Instruction: Printing all the contents of a tensor

Input:
I came across this PyTorch tutorial (in neural_networks_tutorial.py) where they construct a simple neural network and run an inference. I would like to print the contents of the entire input tensor for debugging purposes. What I get when I try to print the tensor is something like this and not the entire tensor:

I saw a similar link for numpy but was not sure about what would work for PyTorch. I can convert it to numpy and may be view it, but would like to avoid the extra overhead. Is there a way for me to print the entire tensor?

Output:
Though I don't suggest to do that, if you want, then

In [18]: torch.set_printoptions(edgeitems=1)

In [19]: a
Out[19]:
tensor([[-0.7698,  ..., -0.1949],
        ...,
        [-0.7321,  ...,  0.8537]])

In [20]: torch.set_printoptions(edgeitems=3)

In [21]: a
Out[21]:
tensor([[-0.7698,  1.3383,  0.5649,  ...,  1.3567,  0.6896, -0.1949],
        [-0.5761, -0.9789, -0.2058,  ..., -0.5843,  2.6311, -0.0008],
        [ 1.3152,  1.8851, -0.9761,  ...,  0.8639, -0.6237,  0.5646],
        ...,
        [ 0.2851,  0.5504, -0.9471,  ...,  0.0688, -0.7777,  0.1661],
        [ 2.9616, -0.8685, -1.5467,  ..., -1.4646,  1.1098, -1.0873],
        [-0.7321,  0.7610,  0.3182,  ...,  2.5859, -0.9709,  0.8537]])


Instruction: How to extend a Loss Function Pytorch

Input:
I would like to create my own custom Loss function as a weighted combination of 3 Loss Function, something similar to:

criterion = torch.nn.CrossEntropyLoss(out1, lbl1) + \
            torch.nn.CrossEntropyLoss(out2, lbl2) + \
            torch.nn.CrossEntropyLoss(out3, lbl3)

I am doing it to address a multi-class multi-label classification problem.
Does it make sense? How to implement correctly such Loss Function in Pytorch?

Thanks

Output:
Your way of approaching the problem seems correct but there's a typo in your code. Here's a fix for that:

loss1 = torch.nn.CrossEntropyLoss()(out1, lbl1)
loss2 = torch.nn.CrossEntropyLoss()(out2, lbl2)
loss3 = torch.nn.CrossEntropyLoss()(out3, lbl3)

final_loss = loss1 + loss2 + loss3

Then you can call .backward on final_loss which should then compute the gradients and backpropagate them.

Also, it's possible to weight each of the component losses where the weights are itself learned during the training process.

You can refer the discussions of combine-multiple-criterions-to-a-loss-function for more information.


Instruction: what does THCudaTensor_data ( and THC in general ) do?

Input:
The program I am inspecting uses pytorch to load weights and cuda code to do the computations with the weights. My understanding of THC library is how tensors are implemented in the backend of pytorch ( and torch, maybe? ).

how is THC implemented ( I would really appreiciate some details if possible )?
what does THCudaTensor_data( THC_state, THCudaTensor* ) do? ( from the way it is used in the code, it seems like it is used to convert pytorch's tensor to an array in cuda. if this is the case, then would the function preserve all elements and the length of the array?)

Output:
I am still not exactly sure of the inner-workings of THCudaTensor_data, but the behaviour that was tripping me up was: for n-dimensional tensor, THCudaTensor_data returns a flattened 1D array of the tensor.

Hope this helps


Instruction: Minimize angle in pytorch

Input:
I'm trying to figure out how to minimize a scalar in pytorch that represents the angle of an axis/angle rotation. My target is a sample set of 3D vectors, my input is the target rotated by a particular axis/angle rotation (plus some gaussian noise). The axis is known and fixed. I want to find the angle using pytorch.

What I have so far is this nn.Module:

class AngleModel(torch.nn.Module):
    def __init__(self):
        super(AngleModel, self).__init__()
        self.angle = nn.Parameter(Variable(torch.Tensor([0.0]), requires_grad=True))
        self.qw = torch.cos(self.angle / 2.)
        self.qx = 0.0
        self.qy = 0.0
        self.qz = torch.sin(self.angle / 2.)

    def forward(self, input):
        matrix_np = transform_from_pq([0, 0, 0, self.qw, self.qx, self.qy, self.qz])
        matrix = torch.from_numpy(matrix_np)
        input_ext = torch.cat((input, torch.ones(input.size(0)).reshape(-1, 1)), 1)
        output = torch.matmul(input_ext.float(), matrix.float())
        return output[:, :3]

However using this in an optimizer fails because model.parameters() returns an empty list (i.e. "empty" generator).

I'm completely new to pytorch. What am I doing wrong?

Relevant code

Initialization:

def _init_model(self):
    self.model = AngleModel()
    self.crit = torch.nn.MSELoss()
    l_rate = 0.01
    print(list(self.model.parameters()))
    self.optim = torch.optim.SGD(self.model.parameters(), lr = l_rate)
    self.epochs = 2000

Learning function:

for epoch in range(self.epochs):
    inputs = torch.from_numpy(normalize(input))
    labels = torch.from_numpy(normalize(target))
    _x = Variable(inputs, requires_grad = True) # without it complains about no grad_fn
    _y = Variable(labels)
    self.optim.zero_grad()

    outputs = self.model.forward(_x)
    loss = self.crit(outputs, _y)
    loss.backward()
    self.optim.step()
    print("Epoch %6d loss %05.3f; %s" % (epoch, loss.data[0], self.model.angle))

Update 1:

Code of the Module updated
Surrounding code added
This at least does not throw any errors anymore but also does not optimize angle

Output:
Here are the necessary changes to the AngleModel class to make it work:

class AngleModel(nn.Module):
    def __init__(self):
        super(AngleModel, self).__init__()
        self.angle = nn.Parameter(torch.tensor(0.0))

    def forward(self, input):
        qw = torch.cos(self.angle / 2.)
        qx = 0.0
        qy = 0.0
        qz = torch.sin(self.angle / 2.)

        matrix = torch.zeros(3, 3)

        matrix[0, 0] = 1. - 2. * qy ** 2 - 2. * qz ** 2
        matrix[1, 1] = 1. - 2. * qx ** 2 - 2. * qz ** 2
        matrix[2, 2] = 1. - 2. * qx ** 2 - 2. * qy ** 2

        matrix[0, 1] = 2. * qx * qy - 2. * qz * qw
        matrix[1, 0] = 2. * qx * qy + 2. * qz * qw

        matrix[0, 2] = 2. * qx * qz + 2 * qy * qw
        matrix[2, 0] = 2. * qx * qz - 2 * qy * qw

        matrix[1, 2] = 2. * qy * qz - 2. * qx * qw
        matrix[2, 1] = 2. * qy * qz + 2. * qx * qw

        output = torch.matmul(input, matrix)
        return output

Among others the main reason that it hadn't been working before was that the graph wasn't consistent due to the intermediate use of numpy. Furthermore the quaternion components qw..qz had been calculated only once in the __init__ method. Their calculation had to move to the forward step too.


Instruction: Does PyTorch seed affect dropout layers?

Input:
I came across the idea of seeding my neural network for reproducible results, and was wondering if pytorch seeding affects dropout layers and what is the proper way to seed my training/testing?

I'm reading the documentation here, and wondering if just placing these lines will be enough?

torch.manual_seed(1)
torch.cuda.manual_seed(1)

Output:
You can easily answer your question with some lines of code:

import torch
from torch import nn

dropout = nn.Dropout(0.5)
torch.manual_seed(9999)
a = dropout(torch.ones(1000))
torch.manual_seed(9999)
b = dropout(torch.ones(1000))
print(sum(abs(a - b)))
# > tensor(0.)

Yes, using manual_seed is enough.


Instruction: XOR neural network does not learn

Input:
I am trying to solve the very simple non-linear problem. It is XOR gate.
I my school knowledge. XOR can be solve by using 2 input nodes, 2 hidden layer nodes. And 1 output. It is binary classification problem.

I generate the 1000 of random integer number it is 0 or 1 and then do backpropagation. But for some unknown reason my network has not learned anything. The training accuracy is constant at 50.

# coding: utf-8
import matplotlib
import torch
import torch.nn as nn
from torch.autograd import Variable

matplotlib.use('TkAgg')  # My buggy OSX 10.13.6 requires this
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from tqdm import tqdm
import random

N = 1000
batch_size = 10
epochs = 40
hidden_size = 2
output_size = 1
lr = 0.1

def return_xor(N):
    tmp_x = []
    tmp_y = []
    for i in range(N):
        a = (random.randint(0, 1) == 1)
        b = (random.randint(0, 1) == 1)
        if (a and not b) or (not a and b):
            q = True
        else:
            q = False
        input_features = (a, b)
        output_class = q
        tmp_x.append(input_features)
        tmp_y.append(output_class)
    return tmp_x, tmp_y

# In[495]:

# Training set
x, y = return_xor(N)
x = torch.tensor(x, dtype=torch.float, requires_grad=True)
y = torch.tensor(y, dtype=torch.float, requires_grad=True)
# Test dataset
x_test, y_test = return_xor(100)
x_test = torch.tensor(x_test)
y_test = torch.tensor(y_test)

class MyDataset(Dataset):
    """Define my own `Dataset` in order to use `Variable` with `autograd`"""

    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __getitem__(self, index):
        return self.x[index], self.y[index]

    def __len__(self):
        return len(self.x)

dataset = MyDataset(x, y)
test_dataset = MyDataset(x_test, y_test)

print(dataset.x.shape)
print(dataset.y.shape)

# Make data iterable by loading to a loader. Shuffle, batch_size kwargs put them here in order to remind I myself
train_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)

print(f"They are {len(train_loader)} batches in the dataset")
shown = 0
for (x, y) in train_loader:
    if shown == 1:
        break
    print(f"{x.shape} {x.dtype}")
    print(f"{y.shape} {y.dtype}")
    shown += 1

class MyModel(nn.Module):
    """
    Binary classification
    2 input nodes
    2 hidden nodes
    1 output node
    """

    def __init__(self, input_size, hidden_size, output_size):
        super().__init__()
        self.fc1 = torch.nn.Linear(input_size, hidden_size)
        self.fc2 = torch.nn.Linear(hidden_size, output_size)
        self.sigmoid = torch.nn.Sigmoid()

    def forward(self, out):
        out = self.fc1(out)
        out = self.fc2(out)
        out = self.sigmoid(out)
        return out

# Create my network
net = MyModel(dataset.x.shape[1], hidden_size, output_size)
CUDA = torch.cuda.is_available()
if CUDA:
    net = net.cuda()
criterion = torch.nn.BCELoss(reduction='elementwise_mean')
optimizer = torch.optim.SGD(net.parameters(), lr=lr)

# Train the network
correct_train = 0
total_train = 0
for epoch in range(epochs):
    for i, (batches, labels) in enumerate(train_loader):
        batcesh = Variable(batches.float())
        labels = Variable(labels.float())
        output = net(batches)  # Forward pass
        optimizer.zero_grad()

        loss = criterion(output, labels.view(10, 1))
        loss.backward()
        optimizer.step()
        total_train += labels.size(0)
        correct_train += (predicted == labels.long()).sum()
        if (i + 1) % 10 == 0:
            print(f"""
                Epoch {epoch+1}/{epochs},
                Iteration {i+1}/{len(dataset)//batch_size},
                Training Loss: {loss.item()},
                Training Accuracy: {100*correct_train/total_train}
              """)

Solution:
I did initialized weight, Adaptive learning rate
https://github.com/elcolie/nnbootcamp/blob/master/Study-XOR.ipynb

Output:
I am not sure what results you are getting, as the code you have posted in the question doesn't work (It gives errors with pytorch 0.4.1 like predicted not defined etc). But syntax issues apart, there are other problems.

Your model is not actually two layer as it does not use non-linearity after the first output. Effectively this is one layer network and to fix that you can modify your model's forward as follows:

def forward(self, out):
    out = torch.nn.functional.relu(self.fc1(out))
    out = self.fc2(out)
    out = self.sigmoid(out)
    return out

You can try sigmoid or tanh non-linearity as well... but the non-linearity is a must. This should fix the problem.

I also see that you are using only 2 hidden units. This might be restrictive and you might want to increase that to something like 5 or 10.


Instruction: How to save LambdaLR scheduler in pytorch with lambda function?

Input:
Running pytorch 0.4.1 with python 3.6 I encountered this problem:
I cannot torch.save my learning rate scheduler because python won't pickle a lambda function:

lambda1 = lambda epoch: epoch // 30
scheduler = LambdaLR(optimizer, lr_lambda=lambda1)
torch.save(scheduler.state_dict(), 'scheduler.pth.tar')

results with an error

PicklingError: Can't pickle <function <lambda> at 0x7f7583fe92f0>:
attribute lookup <lambda> on __main__ failed

How can I save my scheduler?

I know that using a proper function instead of a lambda function for lambda1 can be saved, but I need a lambda function because I want to be able to control this function when it is defined (for instance I want to be able to change the fixed 30 in the denominator).
How can this be done and still allows me to save the scheduler?

Output:
If one wishes to stay with default behavior of torch.save and torch.load, the lambda function can be replaced with a class, for example:

class LRPolicy(object):
    def __init__(self, rate=30):
        self.rate = rate

    def __call__(self, epoch):
        return epoch // self.rate

The scheduler is now

scheduler = LambdaLR(optimizer, lr_lambda=LRPolicy(rate=30))

Now the scheduler can be torch.saveed and torch.load without alternating the pickling module.


Instruction: PyTorch RuntimeError Invalid argument 2 of size

Input:
I am experimenting with a neural network (PyTorch) and I get this error.

  RuntimeError: invalid argument 2: size '[32 x 9216]' is invalid for input with 8192 elements at /pytorch/aten/src/TH/THStorage.cpp:84

My task is about image classification with AlexNet and I have backtracked the error to be the size of the images supplied to the neural network. My question is, given the network architecture with its parameters, how does one determine the correct image size required by the network?

As per my code below, I first transform the training images before feeding into the neural network. But I noticed the neural network can only accept the size of 224 and or else it gives the error above. For instance, my instinct was to apply transforms.RandomResizedCrop of size 64 but apparently this is wrong. Is there a formula to determine the size required?

Code

# transformation to be done on images
transform_train = transforms.Compose([
    transforms.RandomResizedCrop(64),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

class AlexNet(nn.Module):

    def __init__(self, num_classes=1000):
        super(AlexNet, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, num_classes),
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), 256 * 6 * 6)
        x = self.classifier(x)
        return x

Output:
I have figured out the algorithm of getting the right input size.

Out = float(((W−F+2P)/S)+1)

where

Out = Output shape
W = Image volume size (image size)
F = Receptive field (filter size)
P = Padding
S = Stride

Factoring in the given network hyperparameters,

The require Image size I need would be

W = (55 - 1) * 4 - 2(2) + 11
  =  223
  ⩰  224


Instruction: Outer sum, etc. in pytorch

Input:
Numpy offers optimized outer operations for any RxR -> R function, like np.multiply.outer or np.subtract.outer, with the behaviour:

>>> np.subtract.outer([6, 5, 4], [3, 2, 1])
array([[3, 4, 5],
       [2, 3, 4],
       [1, 2, 3]])

Pytorch  does not seem to offer such a feature (or I have missed it).
What is the best / usual / fastest / cleanest way to do so with torch tensors?

Output:
Per the documenation:

  Many PyTorch operations support NumPy Broadcasting Semantics.

An outer subtraction is a broadcasted subtraction from a 2d array to a 1d array, so essentially you can reshape the first array to (3, 1) and then subtract the second array from it:

x = torch.Tensor([6, 5, 4])
y = torch.Tensor([3, 2, 1])

x.reshape(-1, 1) - y
#tensor([[3., 4., 5.],
#        [2., 3., 4.],
#        [1., 2., 3.]])


Instruction: MemoryError when attempting to create a docker image with Torch/PyTorch

Input:
I am following this tutorial to create a Docker image for a flask application. The application depends on Torch/PyTorch. As such, my requirements.txt file looks like the following.

flask
flask-cors
pytorch
torchvision
pandas

My Dockerfile then looks like the following.

FROM ubuntu:latest
LABEL My Company "info@mycompany.com"
RUN apt-get update -y
RUN apt-get install -y python-pip python-dev build-essential
COPY . /app
WORKDIR /app
RUN pip install -vvv --no-cache-dir -r requirements.txt
ENTRYPOINT ["python3"]
CMD ["app.py"]

When I type in the command docker build -t flask-sample-one:latest . I get the following error message (which happens when pip is trying to install torch) on my Mac.

Exception:
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/pip/basecommand.py", line 215, in main
    status = self.run(options, args)
  File "/usr/lib/python2.7/dist-packages/pip/commands/install.py", line 353, in run
    wb.build(autobuilding=True)
  File "/usr/lib/python2.7/dist-packages/pip/wheel.py", line 749, in build
    self.requirement_set.prepare_files(self.finder)
  File "/usr/lib/python2.7/dist-packages/pip/req/req_set.py", line 380, in prepare_files
    ignore_dependencies=self.ignore_dependencies))
  File "/usr/lib/python2.7/dist-packages/pip/req/req_set.py", line 620, in _prepare_file
    session=self.session, hashes=hashes)
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 821, in unpack_url
    hashes=hashes
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 659, in unpack_http_url
    hashes)
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 882, in _download_http_url
    _download_url(resp, link, content_file, hashes)
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 603, in _download_url
    hashes.check_against_chunks(downloaded_chunks)
  File "/usr/lib/python2.7/dist-packages/pip/utils/hashes.py", line 46, in check_against_chunks
    for chunk in chunks:
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 571, in written_chunks
    for chunk in chunks:
  File "/usr/lib/python2.7/dist-packages/pip/utils/ui.py", line 139, in iter
    for x in it:
  File "/usr/lib/python2.7/dist-packages/pip/download.py", line 560, in resp_read
    decode_content=False):
  File "/usr/share/python-wheels/urllib3-1.22-py2.py3-none-any.whl/urllib3/response.py", line 436, in stream
    data = self.read(amt=amt, decode_content=decode_content)
  File "/usr/share/python-wheels/urllib3-1.22-py2.py3-none-any.whl/urllib3/response.py", line 384, in read
    data = self._fp.read(amt)
  File "/usr/share/python-wheels/CacheControl-0.11.7-py2.py3-none-any.whl/cachecontrol/filewrapper.py", line 63, in read
    self._close()
  File "/usr/share/python-wheels/CacheControl-0.11.7-py2.py3-none-any.whl/cachecontrol/filewrapper.py", line 50, in _close
    self.__callback(self.__buf.getvalue())
  File "/usr/share/python-wheels/CacheControl-0.11.7-py2.py3-none-any.whl/cachecontrol/controller.py", line 275, in cache_response
    self.serializer.dumps(request, response, body=body),
  File "/usr/share/python-wheels/CacheControl-0.11.7-py2.py3-none-any.whl/cachecontrol/serialize.py", line 55, in dumps
    "body": _b64_encode_bytes(body),
  File "/usr/share/python-wheels/CacheControl-0.11.7-py2.py3-none-any.whl/cachecontrol/serialize.py", line 12, in _b64_encode_bytes
    return base64.b64encode(b).decode("ascii")
MemoryError
The command '/bin/sh -c pip install -vvv -r requirements.txt' returned a non-zero code: 2

I am not sure what the MemoryError is coming from. I tried to build the docker image on a Macbook Pro with 16 GB of RAM. Any ideas on what's going on or how to create a docker image with Python?

On an AWS EC2 instance, I get the following error.

Running setup.py install for pytorch: started
    Running command /usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-SeBh33/pytorch/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-5jxyuS-record/install-record.txt --single-version-externally-managed --compile
    Traceback (most recent call last):
      File "", line 1, in
      File "/tmp/pip-build-SeBh33/pytorch/setup.py", line 13, in
        raise Exception(message)
    Exception: You should install pytorch from http://pytorch.org
    Running setup.py install for pytorch: finished with status 'error'
Cleaning up...
  Removing source in /tmp/pip-build-SeBh33/pytorch
Command "/usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-SeBh33/pytorch/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-5jxyuS-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /tmp/pip-build-SeBh33/pytorch/
Exception information:
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/pip/basecommand.py", line 215, in main
    status = self.run(options, args)
  File "/usr/lib/python2.7/dist-packages/pip/commands/install.py", line 360, in run
    prefix=options.prefix_path,
  File "/usr/lib/python2.7/dist-packages/pip/req/req_set.py", line 784, in install
    **kwargs
  File "/usr/lib/python2.7/dist-packages/pip/req/req_install.py", line 878, in install
    spinner=spinner,
  File "/usr/lib/python2.7/dist-packages/pip/utils/__init__.py", line 725, in call_subprocess
    % (command_desc, proc.returncode, cwd))
InstallationError: Command "/usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-SeBh33/pytorch/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-5jxyuS-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /tmp/pip-build-SeBh33/pytorch/
The command '/bin/sh -c pip install -vvv --no-cache-dir -r requirements.txt' returned a non-zero code: 1

Any ideas on what's going on? I am simply trying to build a docker image with those few dependencies so I can serve a PyTorch model from a flask application. I'm not sure if there's a tutorial out there on doing this (couldn't find one yet).

Output:
If anyone is interested, I was able to get PyTorch installed into a docker container as follows.

I modified the requirements.txt to look like the following.

flask
flask-cors
pandas

I then modified Dockerfile to look like the following. It's quirky and does not follow the conventional way of install Python packages with pip, but it works. Note that I have to install Python 3, followed by PyTorch wheel, and then finally with a pip install of torchvision.

FROM ubuntu:latest
LABEL My Company "info@mycompany.com"
RUN apt-get update -y
RUN apt-get install -y python-pip python-dev build-essential
RUN apt-get install -y python3.6
RUN apt-get install -y python3-pip
COPY . /app
WORKDIR /app
RUN pip3 install -vvv --no-cache-dir -r requirements.txt
RUN pip3 install http://download.pytorch.org/whl/cpu/torch-0.4.1-cp36-cp36m-linux_x86_64.whl
RUN pip3 install torchvision
ENTRYPOINT ["python3"]
CMD ["app.py"]


Instruction: Odd behaviour of "out" argument in torch.empty()

Input:
I have a sample tensor like this:

In [137]: x = x.new_ones((5, 3), dtype=torch.double)
In [138]: x
Out[138]:
tensor([[1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.]], dtype=torch.float64)

Now, I want to free the memory of this tensor by overwriting the contents using torch.empty() which takes an out argument.

In [139]: torch.empty((5, 3), out=x)
Out[139]:
tensor([[1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.]], dtype=torch.float64)

However, the values in the original tensor x still remains the same. If this is the case, then what is the purpose of this keyword argument out in torch.empty? What am I missing here?

Output:
Here's the C++ implementation of empty with an out param from the source code.

Tensor& empty_out(Tensor& result, IntList size) {
  if (result.is_sparse()) {
    result.sparse_resize_and_clear_(size, size.size(), 0);
  } else {
    result.resize_(size);
  }
  return result;
}

So for dense tensors all it does is resize the tensor appropriately - in your case the size is the same.

In [21]: x = torch.ones((5, 3), dtype=torch.double)

In [22]: torch.empty((2, 3), out=x)
Out[22]:
tensor([[1., 1., 1.],
        [1., 1., 1.]], dtype=torch.float64)

In [23]: torch.empty((2, 8), out=x)
Out[23]:
tensor([[ 1.0000e+00,  1.0000e+00,  1.0000e+00,  1.0000e+00,  1.0000e+00,
          1.0000e+00,  1.0000e+00,  1.0000e+00],
        [ 1.0000e+00,  1.0000e+00,  1.0000e+00,  1.0000e+00,  1.0000e+00,
          1.0000e+00,  1.0000e+00, 4.6631e-310]], dtype=torch.float64)

First of all, empty doesn't free memory - it only cares about allocation of a tensor of an appropriate size. In your case such a tensor has already been allocated, so empty has nothing to do.. it is not going to go allocate a new empty tensor somewhere else in memory. In the second empty example above we are forced to allocate for a tensor with a larger size (2 * 8 = 16 compared to 5 * 3 = 15), and we can see the last element in this empty array is garbage, since it is beyond the contiguous memory block that had previously been initialized. empty won't force-clear your whole tensor to 0 or something like that because again, it is uninitialized data.


Instruction: Setting custom kernel for CNN in pytorch

Input:
Is there a way to specify our own custom kernel values for a convolution neural network in pytorch? Something like kernel_initialiser in tensorflow? Eg. I want a 3x3 kernel in nn.Conv2d with initialization so that it acts as a identity kernel -

0 0 0
0 1 0
0 0 0

(this will effectively return the same output as my input in the very first iteration)

My non-exhaustive research on the subject -

I could use nn.init but it only has some pre-defined kernel initialisaition values.

I tried to follow the discussion on their official thread but it doesn't suit my needs.

I might have missed something in my research please feel free to point out.

Output:
Thanks to ptrblck I was able to solve it.
I can define a new convolution layer as conv and as per the example I can set the identity kernel using -

weights = ch.Tensor([[0, 0, 0], [0, 1, 0], [0, 0, 0]]).unsqueeze(0).unsqueeze(0)
weights.requires_grad = True

conv = nn.Conv2d(1, 1, kernel_size=3, stride=1, padding=1, bias=False)

with ch.no_grad():
    conv.weight = nn.Parameter(weights)

I can then continue to use conv as my regular nn.Conv2d layer.


Instruction: How to get the output from a specific layer from a PyTorch model?

Input: How to extract the features from a specific layer from a pre-trained PyTorch model (such as ResNet or VGG), without doing a forward pass again?

Output:
New answer
Edit: there's a new feature in torchvision v0.11.0 that allows extracting features.
For example, if you wanna extract features from the layer layer4.2.relu_2, you can do like:
import torch
from torchvision.models import resnet50
from torchvision.models.feature_extraction import create_feature_extractor

x = torch.rand(1, 3, 224, 224)

model = resnet50()

return_nodes = {
    "layer4.2.relu_2": "layer4"
}
model2 = create_feature_extractor(model, return_nodes=return_nodes)
intermediate_outputs = model2(x)

Old answer
You can register a forward hook on the specific layer you want. Something like:
def some_specific_layer_hook(module, input_, output):
    pass  # the value is in 'output'

model.some_specific_layer.register_forward_hook(some_specific_layer_hook)

model(some_input)

For example, to obtain the res5c output in ResNet, you may want to use a nonlocal variable (or global in Python 2):
res5c_output = None

def res5c_hook(module, input_, output):
    nonlocal res5c_output
    res5c_output = output

resnet.layer4.register_forward_hook(res5c_hook)

resnet(some_input)

# Then, use `res5c_output`.


Instruction: How to save and use a trained neural network developed in PyTorch / TensorFlow / Keras?

Input: Are there ways to save a model after training and sharing just the model with others? Like a regular script? Since the network is a collection of float matrices, is it possible to just extract these trained weights and run it on new data to make predictions, instead of requiring the users to install these frameworks too? I am new to these frameworks and will make any clarifications as needed.

Output:
PyTorch: As explained in this post, you can save a model's parameters as a dictionary, or load a dictionary to set your model's parameters.
You can also save/load a PyTorch model as an object.
Both procedures require the user to have at least one tensor computation framework installed, e.g. for efficient matrix multiplication.