Instruction: Issues converting Keras code into PyTorch code (shaping)

Input:
I have some keras code that I need to convert to Pytorch. I am new to pytorch and I am having trouble wrapping my head around how to take in input the same way that I did in keras. I have spent many hours on this any tips or help is very appreciated.

Here is the keras code I am dealing with. The input shape is (5000,1)

    def build(input_shape, classes):
        model = Sequential()

        filter_num = ['None',32,64,128,256]
        kernel_size = ['None',8,8,8,8]
        conv_stride_size = ['None',1,1,1,1]
        pool_stride_size = ['None',4,4,4,4]
        pool_size = ['None',8,8,8,8]

        # Block1
        model.add(Conv1D(filters=filter_num[1], kernel_size=kernel_size[1], input_shape=input_shape,
                         strides=conv_stride_size[1], padding='same',
                         name='block1_conv1'))
        model.add(BatchNormalization(axis=-1))
        model.add(ELU(alpha=1.0, name='block1_adv_act1'))
        model.add(Conv1D(filters=filter_num[1], kernel_size=kernel_size[1],
                         strides=conv_stride_size[1], padding='same',
                         name='block1_conv2'))
        model.add(BatchNormalization(axis=-1))
        model.add(ELU(alpha=1.0, name='block1_adv_act2'))
        model.add(MaxPooling1D(pool_size=pool_size[1], strides=pool_stride_size[1],
                               padding='same', name='block1_pool'))
        model.add(Dropout(0.1, name='block1_dropout'))

        # Block 2
        model.add(Conv1D(filters=filter_num[2], kernel_size=kernel_size[2],
                         strides=conv_stride_size[2], padding='same',
                         name='block2_conv1'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block2_act1'))

        model.add(Conv1D(filters=filter_num[2], kernel_size=kernel_size[2],
                         strides=conv_stride_size[2], padding='same',
                         name='block2_conv2'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block2_act2'))
        model.add(MaxPooling1D(pool_size=pool_size[2], strides=pool_stride_size[3],
                               padding='same', name='block2_pool'))
        model.add(Dropout(0.1, name='block2_dropout'))

        # Block 3
        model.add(Conv1D(filters=filter_num[3], kernel_size=kernel_size[3],
                         strides=conv_stride_size[3], padding='same',
                         name='block3_conv1'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block3_act1'))
        model.add(Conv1D(filters=filter_num[3], kernel_size=kernel_size[3],
                         strides=conv_stride_size[3], padding='same',
                         name='block3_conv2'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block3_act2'))
        model.add(MaxPooling1D(pool_size=pool_size[3], strides=pool_stride_size[3],
                               padding='same', name='block3_pool'))
        model.add(Dropout(0.1, name='block3_dropout'))

        # Block 4
        model.add(Conv1D(filters=filter_num[4], kernel_size=kernel_size[4],
                         strides=conv_stride_size[4], padding='same',
                         name='block4_conv1'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block4_act1'))
        model.add(Conv1D(filters=filter_num[4], kernel_size=kernel_size[4],
                         strides=conv_stride_size[4], padding='same',
                         name='block4_conv2'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='block4_act2'))
        model.add(MaxPooling1D(pool_size=pool_size[4], strides=pool_stride_size[4],
                               padding='same', name='block4_pool'))
        model.add(Dropout(0.1, name='block4_dropout'))

        # FC #1
        model.add(Flatten(name='flatten'))
        model.add(Dense(512, kernel_initializer=glorot_uniform(seed=0), name='fc1'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='fc1_act'))

        model.add(Dropout(0.7, name='fc1_dropout'))

        #FC #2
        model.add(Dense(512, kernel_initializer=glorot_uniform(seed=0), name='fc2'))
        model.add(BatchNormalization())
        model.add(Activation('relu', name='fc2_act'))

        model.add(Dropout(0.5, name='fc2_dropout'))

        # Classification
        model.add(Dense(classes, kernel_initializer=glorot_uniform(seed=0), name='fc3'))
        model.add(Activation('softmax', name="softmax"))
        return model

Here are the results of model.summary() from the keras code

Layer (type)                 Output Shape              Param #
=================================================================
block1_conv1 (Conv1D)        (None, 5000, 32)          288
_________________________________________________________________
batch_normalization_1 (Batch (None, 5000, 32)          128
_________________________________________________________________
block1_adv_act1 (ELU)        (None, 5000, 32)          0
_________________________________________________________________
block1_conv2 (Conv1D)        (None, 5000, 32)          8224
_________________________________________________________________
batch_normalization_2 (Batch (None, 5000, 32)          128
_________________________________________________________________
block1_adv_act2 (ELU)        (None, 5000, 32)          0
_________________________________________________________________
block1_pool (MaxPooling1D)   (None, 1250, 32)          0
_________________________________________________________________
block1_dropout (Dropout)     (None, 1250, 32)          0
_________________________________________________________________
block2_conv1 (Conv1D)        (None, 1250, 64)          16448
_________________________________________________________________
batch_normalization_3 (Batch (None, 1250, 64)          256
_________________________________________________________________
block2_act1 (Activation)     (None, 1250, 64)          0
_________________________________________________________________
block2_conv2 (Conv1D)        (None, 1250, 64)          32832
_________________________________________________________________
batch_normalization_4 (Batch (None, 1250, 64)          256
_________________________________________________________________
block2_act2 (Activation)     (None, 1250, 64)          0
_________________________________________________________________
block2_pool (MaxPooling1D)   (None, 313, 64)           0
_________________________________________________________________
block2_dropout (Dropout)     (None, 313, 64)           0
_________________________________________________________________
block3_conv1 (Conv1D)        (None, 313, 128)          65664
_________________________________________________________________
batch_normalization_5 (Batch (None, 313, 128)          512
_________________________________________________________________
block3_act1 (Activation)     (None, 313, 128)          0
_________________________________________________________________
block3_conv2 (Conv1D)        (None, 313, 128)          131200
_________________________________________________________________
batch_normalization_6 (Batch (None, 313, 128)          512
_________________________________________________________________
block3_act2 (Activation)     (None, 313, 128)          0
_________________________________________________________________
block3_pool (MaxPooling1D)   (None, 79, 128)           0
_________________________________________________________________
block3_dropout (Dropout)     (None, 79, 128)           0
_________________________________________________________________
block4_conv1 (Conv1D)        (None, 79, 256)           262400
_________________________________________________________________
batch_normalization_7 (Batch (None, 79, 256)           1024
_________________________________________________________________
block4_act1 (Activation)     (None, 79, 256)           0
_________________________________________________________________
block4_conv2 (Conv1D)        (None, 79, 256)           524544
_________________________________________________________________
batch_normalization_8 (Batch (None, 79, 256)           1024
_________________________________________________________________
block4_act2 (Activation)     (None, 79, 256)           0
_________________________________________________________________
block4_pool (MaxPooling1D)   (None, 20, 256)           0
_________________________________________________________________
block4_dropout (Dropout)     (None, 20, 256)           0
_________________________________________________________________
flatten (Flatten)            (None, 5120)              0
_________________________________________________________________
fc1 (Dense)                  (None, 512)               2621952
_________________________________________________________________
batch_normalization_9 (Batch (None, 512)               2048
_________________________________________________________________
fc1_act (Activation)         (None, 512)               0
_________________________________________________________________
fc1_dropout (Dropout)        (None, 512)               0
_________________________________________________________________
fc2 (Dense)                  (None, 512)               262656
_________________________________________________________________
batch_normalization_10 (Batc (None, 512)               2048
_________________________________________________________________
fc2_act (Activation)         (None, 512)               0
_________________________________________________________________
fc2_dropout (Dropout)        (None, 512)               0
_________________________________________________________________
fc3 (Dense)                  (None, 101)               51813
_________________________________________________________________
softmax (Activation)         (None, 101)               0
=================================================================
Total params: 3,985,957
Trainable params: 3,981,989
Non-trainable params: 3,968

Here is what I have made in pytorch

class model(torch.nn.Module):
    def __init__(self, input_channels, kernel_size, stride, pool_kernel, pool_stride, dropout_p, dropout_inplace=False):
        super(model, self).__init__()
        self.encoder = nn.Sequential(
            BasicBlock1(input_channels, kernel_size, stride, pool_kernel, pool_stride, dropout_p),
            BasicBlock(input_channels//4, kernel_size, stride, pool_kernel, pool_stride, dropout_p),
            BasicBlock(input_channels//16, kernel_size, stride, pool_kernel, pool_stride, dropout_p),
            BasicBlock(input_channels//16//4, kernel_size, stride, pool_kernel, pool_stride, dropout_p)
        )

        self.decoder = nn.Sequential(
            nn.Linear(5120, 512),
            nn.BatchNorm1d(512),
            nn.ReLU(),
            nn.Dropout(p=dropout_p, inplace=dropout_inplace),
            nn.Linear(512, 512),
            nn.BatchNorm1d(512),
            nn.ReLU(),
            nn.Dropout(p=dropout_p, inplace=dropout_inplace),
            nn.Linear(512, 101),
            nn.Softmax(dim=101)
        )
    def forward(self, x):
        x = self.encoder(x)

        x = x.view(x.size(0), -1)  # flatten

        x = self.decoder(x)
        return x

def BasicBlock(input_channels, kernel_size, stride, pool_kernel, pool_stride, dropout_p, dropout_inplace=False):
    return nn.Sequential(
        nn.Conv1d(in_channels=input_channels, out_channels=input_channels, kernel_size=kernel_size, stride=stride,
                  padding=get_pad_size(input_channels, input_channels, kernel_size)),
        nn.BatchNorm1d(32),
        nn.ReLU(),
        nn.Conv1d(in_channels=input_channels, out_channels=input_channels, kernel_size=kernel_size, stride=stride,
                  padding=get_pad_size(input_channels, input_channels, kernel_size)),
        nn.BatchNorm1d(32),
        nn.ReLU(),
        nn.MaxPool1d(kernel_size=pool_kernel, stride=pool_stride,
                     padding=get_pad_size(input_channels, input_channels/4, kernel_size)),
        nn.Dropout(p=dropout_p, inplace=dropout_inplace)
    )

def BasicBlock1(input_channels, kernel_size, stride, pool_kernel, pool_stride, dropout_p, dropout_inplace=False):
    return nn.Sequential(
        nn.Conv1d(in_channels=1, out_channels=input_channels, kernel_size=kernel_size, stride=stride,
                  padding=get_pad_size(input_channels, input_channels, kernel_size)),
        nn.BatchNorm1d(32),
        nn.ReLU(),
        nn.Conv1d(in_channels=input_channels, out_channels=input_channels, kernel_size=kernel_size, stride=stride,
                  padding=get_pad_size(input_channels, input_channels, kernel_size)),
        nn.BatchNorm1d(32),
        nn.ReLU(),
        nn.MaxPool1d(kernel_size=pool_kernel, stride=pool_stride,
                     padding=get_pad_size(input_channels, input_channels/4, kernel_size)),
        nn.Dropout(p=dropout_p, inplace=dropout_inplace)
    )

def get_pad_size(input_shape, output_shape, kernel_size, stride=1, dilation=1):
    """
    Gets the right padded needed to maintain same shape in the conv layers
    BEWARE: works only on odd size kernel size
    :param input_shape: the input shape to the conv layer
    :param output_shape: the desired output shape of the conv layer
    :param kernel_size: the size of the kernel window, has to be odd
    :param stride: Stride of the convolution
    :param dilation: Spacing between kernel elements
    :return: the appropriate pad size for the needed configuration
    :Author: Aneesh
    """

    if kernel_size % 2 == 0:
        raise ValueError(
            "Kernel size has to be odd for this function to work properly. Current Value is %d." % kernel_size)

    return (int((output_shape * stride - stride + kernel_size - input_shape + (kernel_size - 1) * (dilation - 1)) / 2))

Lastly here is the model summary for what my pytorch model creates

model(
  (encoder): Sequential(
    (0): Sequential(
      (0): Conv1d(1, 5000, kernel_size=(7,), stride=(1,), padding=(3,))
      (1): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): Conv1d(5000, 5000, kernel_size=(7,), stride=(1,), padding=(3,))
      (4): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (5): ReLU()
      (6): MaxPool1d(kernel_size=8, stride=4, padding=-1872, dilation=1, ceil_mode=False)
      (7): Dropout(p=0.1)
    )
    (1): Sequential(
      (0): Conv1d(1250, 1250, kernel_size=(7,), stride=(1,), padding=(3,))
      (1): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): Conv1d(1250, 1250, kernel_size=(7,), stride=(1,), padding=(3,))
      (4): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (5): ReLU()
      (6): MaxPool1d(kernel_size=8, stride=4, padding=-465, dilation=1, ceil_mode=False)
      (7): Dropout(p=0.1)
    )
    (2): Sequential(
      (0): Conv1d(312, 312, kernel_size=(7,), stride=(1,), padding=(3,))
      (1): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): Conv1d(312, 312, kernel_size=(7,), stride=(1,), padding=(3,))
      (4): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (5): ReLU()
      (6): MaxPool1d(kernel_size=8, stride=4, padding=-114, dilation=1, ceil_mode=False)
      (7): Dropout(p=0.1)
    )
    (3): Sequential(
      (0): Conv1d(78, 78, kernel_size=(7,), stride=(1,), padding=(3,))
      (1): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): ReLU()
      (3): Conv1d(78, 78, kernel_size=(7,), stride=(1,), padding=(3,))
      (4): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (5): ReLU()
      (6): MaxPool1d(kernel_size=8, stride=4, padding=-26, dilation=1, ceil_mode=False)
      (7): Dropout(p=0.1)
    )
  )
  (decoder): Sequential(
    (0): Linear(in_features=5120, out_features=512, bias=True)
    (1): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU()
    (3): Dropout(p=0.1)
    (4): Linear(in_features=512, out_features=512, bias=True)
    (5): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (6): ReLU()
    (7): Dropout(p=0.1)
    (8): Linear(in_features=512, out_features=101, bias=True)
    (9): Softmax()
  )
)

Output:
I think your fundamental problem is that you confuse in_channels and out_channels with Keras shapes. Let's just take the first convolutional layer as an example. In Keras you have:

Conv1D(filters=32, kernel_size=8, input_shape=(5000,1), strides=1, padding='same')

The PyTorch equivalent should be (changing the kernel size to 7 like you did, we'll come back to it later):

nn.Conv1d(in_channels=1, out_channels=32, kernel_size=7, stride=1, padding=3) # different kernel size

Note that you don't need to give the shape of your input sequence for pytorch. Now let's see how it compares to what you did:

nn.Conv1d(in_channels=1, out_channels=5000, kernel_size=7, stride=1, padding=0) # note padding

You just created a huge network. While the correct implementation produces an output of [b, 32, 5000] where b is the batch size, your output is [b, 5000, 5000].

Hope this example helps you to correct the rest of your implementation.

Finally, some notes on replicating same padding in pytorch. With even kernel sizes, to preserve the size of your input you need asymmetric padding. This I think might not be available when you create the layer. I see you instead changed the kernel size to 7, but it can actually be done with the original kernel size of 8. You can use padding in your forward() function to create the required asymmetric padding.

layer = nn.Conv1d(in_channels=1, out_channels=32, kernel_size=8, stride=1, padding=0) # layer without padding
x = torch.empty(1, 1, 5000).normal_()  # random input

# forward run
x_padded = torch.nn.functional.pad(x, (3,4))
y = layer(x_padded).shape
print(y.shape)  # torch.Size([1, 32, 5000])


Instruction: Preventing PyTorch Dataset iteration from exceeding length of dataset

Input:
I am using a custom PyTorch Dataset with the following:

class ImageDataset(Dataset):
    def __init__(self, input_dir, input_num, input_format, transform=None):
        self.input_num = input_num
        # etc
    def __len__ (self):
        return self.input_num
    def __getitem__(self,idx):
        targetnum = idx % self.input_num
        # etc

However, when I iterate over this dataset, iteration loops back to the start of the dataset instead of terminating at the end of the dataset. This effectively becomes an infinite loop in the iterator, with the epoch print statement never occurring for subsequent epochs.

train_dataset=ImageDataset(input_dir = 'path/to/directory',
                           input_num = 300, input_format = "mask") # Size 300
num_epochs = 10
for epoch in range(num_epochs):
    print("EPOCH " + str(epoch+1) + "\n")
    num = 0
    for data in train_dataset:
        print(num, end=" ")
        num += 1
        # etc

Print output (... for values in between):

EPOCH 1
0 1 2 3 4 5 6 7 ... 298 299 300 301 302 303 304 305 ... 597 598 599 600 601 602 603 604 ...

Why is the basic iteration over the Dataset continuing past the defined __len__ of the DataSet, and how can I ensure that iteration over the dataset terminates after hitting the length of the dataset when using this method (or is manually iterating over the range of the dataset length the only solution)?

Thank you.

Output:
Dataset class doesn't have implemented StopIteration signal.

  The for loop listens for StopIteration. The purpose of the for statement is to loop over the sequence provided by an iterator and the exception is used to signal that the iterator is now done...

More: Why does next raise a 'StopIteration', but 'for' do a normal return? | The Iterator Protocol


Instruction: Cuda Runtime/Driver incompatibility in docker container

Input:
I'm trying to run this simple line of code in a docker container that comes with Pytorch.

import torch
torch.cuda.set_device(0)

I get this error:

RuntimeError: cuda runtime error (35) : CUDA driver version is insufficient for CUDA runtime version at torch/csrc/cuda/Module.cpp:32

Running torch.cuda.is_available() returns False.

The host machine has the most up-to-date Nvidia drivers. Pytorch ships with Cuda, so there should be no incompatibility issues.

What could cause this problem?

Edit:
@Patel Sunil's answer to this question answers my question, but I didn't come across this question in my search because their question is broad, while my question is specific to the cuda runtime/driver error. I posted this as a separate question for those who come across this error but don't know what it is a symptom of (namely, forgetting to use nvidia-docker).

Output: The problem was that I was running the container with docker, not nvidia-docker. Running the docker container with nvidia-docker fixed the problem.


Instruction: Compute Optical Flow corresponding to data in the torch.utils.data.DataLoader

Input:
I have built a CNN model for action recognition in videos in PyTorch. I'm loading the data for training using the torch dataloader module.

train_loader = torch.utils.data.DataLoader(
            training_data,
            batch_size=8,
            shuffle=True,
            num_workers=4,
            pin_memory=True)

And then passing the train_loader for training the model.

train_epoch(i, train_loader, action_detect_model, criterion, optimizer, opt,
                        train_logger, train_batch_logger)

Now I want to add an additional path which will take the corresponding optical flow of the video frames. To calculate the optical flow I'm using cv2.calcOpticalFlowFarneback.  But the problem is that I'm not sure how to get the images corresponding to the data in the train data loader tensor as they will be shuffled. I don't want to pre-compute the optical flow as the storage requirement will be huge (each frame takes 600 kBs).

Output:
You have to use an own data loader class to compute optical flow on the fly. The idea is that this class get a list of filename tuples (curr image, next image) containing the current and next frame filenames of the video sequence instead of simple filename list. This allows to get the correct image pairs after suffling the filename list.
The following code gives you a very simple example implementaton:

from torch.utils.data import Dataset
import cv2
import numpy as np

class FlowDataLoader(Dataset):
def __init__(self,
             filename_tuples):

    random.shuffle(filename_tuples)
    self.lines = filename_tuples

def __getitem__(self, index):
    img_filenames = self.lines[index]
    curr_img = cv2.cvtColor(cv2.imread(img_filenames[0]), cv2.BGR2GRAY)
    next_img = cv2.cvtColor(cv2.imread(img_filenames[1]), cv2.BGR2GRAY)
    flow = cv2.calcOpticalFlowFarneback(curr_img, next_img, ... [parameter])

    # code for loading the class label
    # label = ...
    #
    # this is a very simple data normalization
    curr_img= curr_img.astype(np.float32) / 255
    next_img = next_img .astype(np.float32) / 255
    # you can return the image and flow seperatly
    return curr_img, flow, label
    # or stacked as follows
    # return np.dstack((curr_img,flow)), label

# at this place you need a function that create a list of training sample filenames
# that look like this
training_filelist = [(img000.png, img001.png),
                     (img001.png, img002.png),
                     (img002.png, img003.png)]

training_data = FlowDataLoader(training_filelist)
train_loader = torch.utils.data.DataLoader(
        training_data,
        batch_size=8,
        shuffle=True,
        num_workers=4,
        pin_memory=True)

This is only a simple example of the FlowDataLoader. Idealy this should be extended so that curr_image output contains normalized rgb values and the optical flow is normalized and clipped too.


Instruction: How to iterate over a group of tensor and pass the elements from each group to a function?

Input:
Suppose you have 3 tensors of the same size:

a = torch.randn(3,3)
a = ([[ 0.1945,  0.8583,  2.6479],
    [-0.1000,  1.2136, -0.3706],
    [-0.0094,  0.4279, -0.6840]])

b = torch.randn(3, 3)
b = ([[-1.1155,  0.2106, -0.2183],
    [ 1.6610, -0.6953,  0.0052],
    [-0.8955,  0.0953, -0.7737]])

c = torch.randn(3, 3)
c = ([[-0.2303, -0.3427, -0.4990],
    [-1.1254,  0.4432,  0.3999],
    [ 0.2489, -0.9459, -0.5576]])

In Lua (torch7), they have this function:

[self] map2(tensor1, tensor2, function(x, xt1, xt2))

which applies the given function to all elements of self.

My questions are:

Is there any similar function in python (pytorch)?
Is there any pythonic method to iterate over the 3 tensors and get the respective elements of each tensor without using for loop and indices?

For example:

0.1945 -1.1155 -0.2303
0.8583 0.2106 -0.3427
2.6479 -0.2183 -0.4990
-0.1000 1.6610 -1.1254
...

Edit_1: I have also tried itertools.zip_longest and zip, but the results are not as I expected as mentioned above

Output:
You can use Python's map function similar to what you have mentioned. Like this:

>>> tensor_list = [torch.tensor([i, i, i]) for i in range(3)]
>>> list(map(lambda x: x**2, tensor_list))
[tensor([0, 0, 0]), tensor([1, 1, 1]), tensor([4, 4, 4])]
>>>

EDIT: For a PyTorch only approach you can use torch.Tensor.apply_ (Note this does the changes in place and doesn't return a new tensor)

>>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
>>> x.apply_(lambda y: y ** 2)
tensor([[ 1,  4,  9],
        [16, 25, 36],
        [49, 64, 81]])
>>>


Instruction: Keras learning rate decay in pytorch

Input:
I have a question concerning learning rate decay in Keras. I need to understand how the option decay works inside optimizers in order to translate it to an equivalent PyTorch formulation.

From the source code of SGD I see that the update is done this way after every batch update:

lr = self.lr * (1. / (1. + self.decay * self.iterations))

Does this mean that after every batch update the lr is updated starting from its value from its previous update or from its initial value? I mean, which of the two following interpretation is the correct one?

lr = lr_0 * (1. / (1. + self.decay * self.iterations))

or

lr = lr * (1. / (1. + self.decay * self.iterations)),

where lr is the lr updated after previous iteration and lr_0 is always the initial learning rate.

If the correct answer is the first one, this would mean that, in my case, the learning rate would decay from 0.001 to just 0.0002 after 100 epochs, whereas in the second case it would decay from 0.001 at around 1e-230 after 70 epochs.

Just to give you some context, I'm working with a CNN for a regression problem from images and I just have to translate Keras code into Pytorch code. So far, with the second of the afore-mentioned interpretations I manage to only always predict the same value, disregarding of batch size and input at test time.

Thanks in advance for your help!

Output:
Based on the implementation in Keras I think your first formulation is the correct one, the one that contain the initial learning rate (note that self.lr is not being updated).

However I think your calculation is probably not correct: since the denominator is the same, and lr_0 >= lr since you are doing decay, the first formulation has to result in a bigger number.

I'm not sure if this decay is available in PyTorch, but you can easily create something similar with torch.optim.lr_scheduler.LambdaLR.

decay = .001
fcn = lambda step: 1./(1. + decay*step)
scheduler = LambdaLR(optimizer, lr_lambda=fcn)

Finally, don't forget that you will need to call .step() explicitly on the scheduler, it's not enough to step your optimizer. Also, most often learning scheduling is only done after a full epoch, not after every single batch, but I see that here you are just recreating Keras behavior.


Instruction: I change the expected object of scalar type float but still got Long in Pytorch

Input:
To do the binary class classification. I use binary cross entropy to be the loss function(nn.BCEloss()), and the units of last layer is one.

Before I put (input, target) into loss function, I cast target from Long to float. Only the final step of the DataLoader comes the error messages, and the error message is as bellows.

"RuntimeError: Expected object of scalar type Float but got scalar type Long for argument #2 'target'"

The DataLoader(I drop the last batch if the batch size is not match) is defined in the code, I'm not sure if there is a correlation with the error.

I have tried to print the type of the target and input(output of the Neural Network), and the type of both variable is float. I put the "type result" and the code bellow.

trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE,
                                          shuffle=True, drop_last=True)
loss_func = nn.BCELoss()

# training
for epoch in range(EPOCH):
    test_loss = 0
    train_loss = 0

    for step, (b_x, b_y) in enumerate(trainloader):        # gives batch data
        b_x = b_x.view(-1, TIME_STEP, 1)              # reshape x to (batch, time_step, input_size)
        print("step: ", step)
        b_x = b_x.to(device)
        print("BEFORE|b_y type: ",b_y.type())
        b_y = b_y.to(device, dtype=torch.float)
        print("AFTER|b_y type: ",b_y.type())
        output = rnn(b_x)                               # rnn output
        print("output type:", output.type())
        loss = loss_func(output, b_y)  # !!!error occurs when trainloader enumerate the final step!!!

        train_loss = train_loss + loss

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

#### type result and the error message####
...
step:  6
BEFORE|b_y type:  torch.LongTensor
AFTER|b_y type:  torch.cuda.FloatTensor
output type: torch.cuda.FloatTensor
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-18-e028fcb6b840> in <module>
     30         b_y = b_y.to(device)
     31         output = rnn(b_x)
---> 32         loss = loss_func(output, b_y)
     33         test_loss = test_loss + loss
     34         rnn.train()

~/venvs/tf1.12/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/venvs/tf1.12/lib/python3.5/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    502     @weak_script_method
    503     def forward(self, input, target):
--> 504         return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
    505
    506

~/venvs/tf1.12/lib/python3.5/site-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
   2025
   2026     return torch._C._nn.binary_cross_entropy(
-> 2027         input, target, weight, reduction_enum)
   2028
   2029

RuntimeError: Expected object of scalar type Float but got scalar type Long for argument #2 'target'

Output:
It appears that the type is correctly being changed, as you state that you observe the change when printing the types and from Pytorch:

Returns a Tensor with the specified device and (optional) dtype. If
dtype is None it is inferred to be self.dtype. When non_blocking,
tries to convert asynchronously with respect to the host if possible,
e.g., converting a CPU Tensor with pinned memory to a CUDA Tensor.
When copy is set, a new Tensor is created even when the Tensor already
matches the desired conversion.

and other methods like
b_y = b_y.to(device).float()

should not be measurably different as , again, .float() is equivalent to .to(..., torch.float32). and .float is equivalent to .float32. Can you verify the type of b_y right before the error is thrown and edit the question? (I would have made this a comment - but I wanted to add more detail. I will try to help when that is provided)


Instruction: What's the fastest way to copy values from one tensor to another in PyTorch?

Input:
I am experimenting with dilation in convolution where I am trying to copy data from one 2D tensor to another 2D tensor using PyTorch. I'm copying values from tensor A to tensor B such that every element of A that is copied into B is surrounded by n zeros.

I have already tried using nested for loops, which is a very naive way. The performance, obviously is quite bad when I'm using large number of grayscale images as input.

for i in range(A.shape[0]):
   for j in range(A.shape[1]):
      B[n+i][n+j] = A[i][j]

Is there anything faster that doesn't need the usage of loops?

Output:
If I understand your question correctly, here is a faster alternative, without any loops:

# sample `n`
In [108]: n = 2

# sample tensor to work with
In [102]: A = torch.arange(start=1, end=5*4 + 1).view(5, -1)

In [103]: A
Out[103]:
tensor([[ 1,  2,  3,  4],
        [ 5,  6,  7,  8],
        [ 9, 10, 11, 12],
        [13, 14, 15, 16],
        [17, 18, 19, 20]])

# our target tensor where we will copy values
# we need to multiply `n` by 2 since there are two axes
In [104]: B = torch.zeros(A.shape[0] + 2*n, A.shape[1] + 2*n)

# copy the values, at the center of the grid
# leaving `n` positions on the surrounding
In [106]: B[n:-n, n:-n] = A

# check whether we did it correctly
In [107]: B
Out[107]:
tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  1.,  2.,  3.,  4.,  0.,  0.],
        [ 0.,  0.,  5.,  6.,  7.,  8.,  0.,  0.],
        [ 0.,  0.,  9., 10., 11., 12.,  0.,  0.],
        [ 0.,  0., 13., 14., 15., 16.,  0.,  0.],
        [ 0.,  0., 17., 18., 19., 20.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])

Another case where n=3

In [118]: n = 3

# we need to multiply `n` by 2 since there are two axes
In [119]: B = torch.zeros(A.shape[0] + 2*n, A.shape[1] + 2*n)

# copy the values, at the center of the grid
# leaving `n` positions on the surrounding
In [120]: B[n:-n, n:-n] = A

In [121]: B
Out[121]:
tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  1.,  2.,  3.,  4.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  5.,  6.,  7.,  8.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  9., 10., 11., 12.,  0.,  0.,  0.],
        [ 0.,  0.,  0., 13., 14., 15., 16.,  0.,  0.,  0.],
        [ 0.,  0.,  0., 17., 18., 19., 20.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])

sanity check with your loop based solution:

In [122]: n = 2
In [123]: B = torch.zeros(A.shape[0] + 2*n, A.shape[1] + 2*n)
In [124]: for i in range(A.shape[0]):
     ...:    for j in range(A.shape[1]):
     ...:       B[n+i][n+j] = A[i][j]
     ...:

In [125]: B
Out[125]:
tensor([[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  1.,  2.,  3.,  4.,  0.,  0.],
        [ 0.,  0.,  5.,  6.,  7.,  8.,  0.,  0.],
        [ 0.,  0.,  9., 10., 11., 12.,  0.,  0.],
        [ 0.,  0., 13., 14., 15., 16.,  0.,  0.],
        [ 0.,  0., 17., 18., 19., 20.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.],
        [ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.]])

timings:

# large sized input tensor
In [126]: A = torch.arange(start=1, end=5000*4 + 1).view(5000, -1)
In [127]: n = 2
In [132]: B = torch.zeros(A.shape[0] + 2*n, A.shape[1] + 2*n)
# loopy solution
In [133]: %%timeit
     ...: for i in range(A.shape[0]):
     ...:    for j in range(A.shape[1]):
     ...:       B[n+i][n+j] = A[i][j]
     ...:
92.1 ms ± 434 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

# clear out `B` again by reinitializing it.
In [128]: B = torch.zeros(A.shape[0] + 2*n, A.shape[1] + 2*n)

In [129]: %timeit B[n:-n, n:-n] = A
49.6 µs ± 239 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

From the above timings, we can see that the vectorized approach is ~200x faster than loop based solution.


Instruction: Why is an OOM happening on my model init()?

Input:
A single line in my model, tr.nn.Linear(hw_flat * num_filters*8, num_fc), is causing an OOM error on initialization of the model. Commenting it out removes the memory issue.

import torch as tr
from layers import Conv2dSame, Flatten

class Discriminator(tr.nn.Module):
    def __init__(self, cfg):
        super(Discriminator, self).__init__()
        num_filters = 64
        hw_flat = int(cfg.hr_resolution[0] / 2**4)**2
        num_fc = 1024

        self.model = tr.nn.Sequential(
            # Channels in, channels out, filter size, stride, padding
            Conv2dSame(cfg.num_channels, num_filters, 3),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters, num_filters, 3, 2),
            tr.nn.BatchNorm2d(num_filters),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters, num_filters*2, 3),
            tr.nn.BatchNorm2d(num_filters*2),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters*2, num_filters*2, 3, 2),
            tr.nn.BatchNorm2d(num_filters*2),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters*2, num_filters*4, 3),
            tr.nn.BatchNorm2d(num_filters*4),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters*4, num_filters*4, 3, 2),
            tr.nn.BatchNorm2d(num_filters*4),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters*4, num_filters*8, 3),
            tr.nn.BatchNorm2d(num_filters*8),
            tr.nn.LeakyReLU(),
            Conv2dSame(num_filters*8, num_filters*8, 3, 2),
            tr.nn.BatchNorm2d(num_filters*8),
            tr.nn.LeakyReLU(),
            Flatten(),
            tr.nn.Linear(hw_flat * num_filters*8, num_fc),
            tr.nn.LeakyReLU(),
            tr.nn.Linear(num_fc, 1),
            tr.nn.Sigmoid()
        )
        self.model.apply(self.init_weights)

    def forward(self, x_in):
        x_out = self.model(x_in)
        return x_out

    def init_weights(self, layer):
        if type(layer) in [tr.nn.Conv2d, tr.nn.Linear]:
            tr.nn.init.xavier_uniform_(layer.weight)

This is strange, as hw_flat = 96*96 = 9216, and num_filters*8 = 512, so hw_flat * num_filters = 4718592, which is the number of parameters in that layer. I have confirmed this calculation as changing the layer to tr.nn.Linear(4718592, num_fc) results in the same output.

To me this makes no sense as dtype=float32, so the expected size of this would be 32*4718592 = 150,994,944 bytes. This is equivalent to about 150mb.

Error message is:

Traceback (most recent call last):
  File "main.py", line 116, in <module>
    main()
  File "main.py", line 112, in main
    srgan = SRGAN(cfg)
  File "main.py", line 25, in __init__
    self.discriminator = Discriminator(cfg).to(device)
  File "/home/jpatts/Documents/ECE/ECE471-SRGAN/models.py", line 87, in __init__
    tr.nn.Linear(hw_flat * num_filters*8, num_fc),
  File "/home/jpatts/.local/lib/python3.6/site-packages/torch/nn/modules/linear.py", line 51, in __init__
    self.weight = Parameter(torch.Tensor(out_features, in_features))
RuntimeError: $ Torch: not enough memory: you tried to allocate 18GB. Buy new RAM! at /pytorch/aten/src/TH/THGeneral.cpp:201

I am only running batch sizes of 1 as well (not that that affects this error), with overall input shape to the network being (1, 3, 1536, 1536), and shape after flatten layer being (1, 4718592).

Why is this happening?

Output:
Your linear layer is quite large - it does, in fact, need at least 18GB of memory. (Your estimate is off for two reasons: (1) a float32 takes 4 bytes of memory, not 32, and (2) you didn't multiply by the output size.)

From the PyTorch documentation FAQs:

  Don’t use linear layers that are too large. A linear layer nn.Linear(m, n) uses O(n*m)
  memory: that is to say, the memory requirements of the weights scales quadratically with
  the number of features. It is very easy to blow through your memory this way (and
  remember that you will need at least twice the size of the weights, since you also need
  to store the gradients.)


Instruction: Loss is 'nan' all the time when training the neural network in PyTorch

Input:
I assigned different  weight_decayfor the parameters, and the training loss and testing loss were all nan.

I printed the prediction_train,loss_train,running_loss_train,prediction_test,loss_test,and running_loss_test ,they were all nan.

And I have checked the data with numpy.any(numpy.isnan(dataset)), it returned False.

If I use optimizer = torch.optim.Adam(wnn.parameters()) rather than assigning different weight_decay for the parameters, there would be no problem.

Could you please tell me how to fix it? Here are the codes, I defined the activation function by myself. Thank you:)

class Morlet(nn.Module):
def __init__(self):
    super(Morlet,self).__init__()
def forward(self,x):
    x=(torch.cos(1.75*x))*(torch.exp(-0.5*x*x))
    return x

morlet=Morlet()

class WNN(nn.Module):
def __init__(self):
    super(WNN,self).__init__()
    self.a1=torch.nn.Parameter(torch.randn(64,requires_grad=True))
    self.b1=torch.nn.Parameter(torch.randn(64,requires_grad=True))
    self.layer1=nn.Linear(30,64,bias=False)
    self.out=nn.Linear(64,1)
def forward(self,x):
    x=self.layer1(x)
    x=(x-self.b1)/self.a1
    x=morlet(x)
    out=self.out(x)
    return out
wnn=WNN()

optimizer = torch.optim.Adam([{'params': wnn.layer1.weight, 'weight_decay':0.01},
                          {'params': wnn.out.weight, 'weight_decay':0.01},
                          {'params': wnn.out.bias, 'weight_decay':0},
                          {'params': wnn.a1, 'weight_decay':0.01},
                          {'params': wnn.b1, 'weight_decay':0.01}])
criterion = nn.MSELoss()

for epoch in range(10):
prediction_test_list=[]
running_loss_train=0
running_loss_test=0
for i,(x1,y1) in enumerate(trainloader):
    prediction_train=wnn(x1)
    #print(prediction_train)
    loss_train=criterion(prediction_train,y1)
    #print(loss_train)
    optimizer.zero_grad()
    loss_train.backward()
    optimizer.step()
    running_loss_train+=loss_train.item()
    #print(running_loss_train)
tr_loss=running_loss_train/train_set_y_array.shape[0]
for i,(x2,y2) in enumerate(testloader):
    prediction_test=wnn(x2)
    #print(prediction_test)
    loss_test=criterion(prediction_test,y2)
    #print(loss_test)
    running_loss_test+=loss_test.item()
    print(running_loss_test)
    prediction_test_list.append(prediction_test.detach().cpu())
ts_loss=running_loss_test/test_set_y_array.shape[0]

print('Epoch {} Train Loss:{}, Test Loss:{}'.format(epoch+1,tr_loss,ts_loss))

test_set_y_array_plot=test_set_y_array*(dataset.max()-dataset.min())+dataset.min()
prediction_test_np=torch.cat(prediction_test_list).numpy()
prediction_test_plot=prediction_test_np*(dataset.max()-dataset.min())+dataset.min()

plt.plot(test_set_y_array_plot.flatten(),'r-',linewidth=0.5,label='True data')
plt.plot(prediction_test_plot,'b-',linewidth=0.5,label='Predicted data')

plt.legend()
plt.show()

print('Finish training')

The output was:

Epoch 1 Train Loss:nan, Test Loss:nan

And there was only the true data on the plot, as the picture shows.

Output: Weight decay applies L2 regularization to the learned parameters, taking a quick glance at your code, you are using the a1 weights as denomenators here x=(x-self.b1)/self.a1 with a weight decay of .01, this could lead to eliminating some of those a1 weights to be zero, and what are the results of a division by zero ?


Instruction: Adam optimizer error: one of the variables needed for gradient computation has been modified by an inplace operation

Input:
I am trying to implement Actor-Critic learning atuomation algorithm that is not same as basic actor-critic algorithm, it's little bit changed.

Anyway, I used Adam optimizer and implemented with pytorch

when i backward TD-error for Critic first, there's no error.
However, i backward loss for Actor, the error occured.

  --------------------------------------------------------------------------- RuntimeError                              Traceback (most recent call
  last)  in
       46             # update Actor Func
       47             optimizer_M.zero_grad()
  ---> 48             loss.backward()
       49             optimizer_M.step()
       50

  ~\Anaconda3\lib\site-packages\torch\tensor.py in backward(self,
  gradient, retain_graph, create_graph)
      100                 products. Defaults to False.
      101         """
  --> 102         torch.autograd.backward(self, gradient, retain_graph, create_graph)
      103
      104     def register_hook(self, hook):

  ~\Anaconda3\lib\site-packages\torch\autograd__init__.py in
  backward(tensors, grad_tensors, retain_graph, create_graph,
  grad_variables)
       88     Variable._execution_engine.run_backward(
       89         tensors, grad_tensors, retain_graph, create_graph,
  ---> 90         allow_unreachable=True)  # allow_unreachable flag
       91
       92

  RuntimeError: one of the variables needed for gradient computation has
  been modified by an inplace operation

above is the content of error

I tried to find inplace operation, but I haven't found in my written code.
I think i don't know how to handle optimizer.

Here is main code:

        for cur_step in range(1):
        action = M_Agent(state, flag)
        next_state, r = env.step(action)

        # calculate TD Error
        TD_error = M_Agent.cal_td_error(r, next_state)

        # calculate Target
        target = torch.FloatTensor([M_Agent.cal_target(TD_error)])
        logit = M_Agent.cal_logit()
        loss = criterion(logit, target)

        # update value Func
        optimizer_M.zero_grad()
        TD_error.backward()
        optimizer_M.step()

        # update Actor Func
        loss.backward()
        optimizer_M.step()

Here is the agent network

    # Actor-Critic Agent
    self.act_pipe = nn.Sequential(nn.Linear(state, 128),
                            nn.ReLU(),
                            nn.Dropout(0.5),
                            nn.Linear(128, 256),
                            nn.ReLU(),
                            nn.Dropout(0.5),
                            nn.Linear(256, num_action),
                            nn.Softmax()
                            )

     self.val_pipe = nn.Sequential(nn.Linear(state, 128),
                            nn.ReLU(),
                            nn.Dropout(0.5),
                            nn.Linear(128, 256),
                            nn.ReLU(),
                            nn.Dropout(0.5),
                            nn.Linear(256, 1)
                            )

      def forward(self, state, flag, test=None):

          temp_action_prob = self.act_pipe(state)
          self.action_prob = self.cal_prob(temp_action_prob, flag)
          self.action = self.get_action(self.action_prob)
          self.value = self.val_pipe(state)

          return self.action

I wanna update each network respectively.

and I wanna know that Basic TD Actor-Critic method uses TD error for loss??
or squared error between r+V(s') and V(s) ?

Output:
I think the problem is that you zero the gradients right before calling backward, after the forward propagation. Note that for automatic differentiation you need the computation graph and the intermediate results that you produce during your forward pass.

So zero the gradients before your TD error and target calculations! And not after you are finished your forward propagation.

    for cur_step in range(1):
    action = M_Agent(state, flag)
    next_state, r = env.step(action)

    optimizer_M.zero_grad()  # zero your gradient here

    # calculate TD Error
    TD_error = M_Agent.cal_td_error(r, next_state)

    # calculate Target
    target = torch.FloatTensor([M_Agent.cal_target(TD_error)])
    logit = M_Agent.cal_logit()
    loss = criterion(logit, target)

    # update value Func
    TD_error.backward()
    optimizer_M.step()

    # update Actor Func
    loss.backward()
    optimizer_M.step()

To answer your second question, the DDPG algorithm for example uses the squared error (see the paper).

Another recommendation. In many cases large parts of the value and policy networks are shared in deep actor-critic agents: you have the same layers up to the last hidden layer, and use a single linear output for value prediction and a softmax layer for the action distribution. This is especially useful if you have high dimensional visual inputs, as it act as sort of a multi-task learning, but nevertheless you can try. (As I see you have a low-dimensional state vector).


Instruction: Should I use softmax as output when using cross entropy loss in pytorch?

Input:
I have a problem with classifying fully connected deep neural net with 2 hidden layers for MNIST dataset in pytorch.
I want to use tanh as activations in both hidden layers, but in the end, I should use softmax.
For the loss, I am choosing nn.CrossEntropyLoss() in PyTOrch, which (as I have found out) does not want to take one-hot encoded labels as true labels, but takes LongTensor of classes instead.
My model is nn.Sequential() and when I am using softmax in the end, it gives me worse results in terms of accuracy on testing data. Why?
import torch
from torch import nn

inputs, n_hidden0, n_hidden1, out = 784, 128, 64, 10
n_epochs = 500
model = nn.Sequential(
    nn.Linear(inputs, n_hidden0, bias=True),
    nn.Tanh(),
    nn.Linear(n_hidden0, n_hidden1, bias=True),
    nn.Tanh(),
    nn.Linear(n_hidden1, out, bias=True),
    nn.Softmax()  # SHOULD THIS BE THERE?
)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.5)

for epoch in range(n_epochs):
    y_pred = model(X_train)
    loss = criterion(y_pred, Y_train)
    print('epoch: ', epoch+1,' loss: ', loss.item())
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

Output:
As stated in the torch.nn.CrossEntropyLoss() doc:

This criterion combines nn.LogSoftmax() and nn.NLLLoss() in one single class.

Therefore, you should not use softmax before.


Instruction: Label Smoothing in PyTorch

Input:
I'm building a ResNet-18 classification model for the Stanford Cars dataset using transfer learning. I would like to implement label smoothing to penalize overconfident predictions and improve generalization.
TensorFlow has a simple keyword argument in CrossEntropyLoss. Has anyone built a similar function for PyTorch that I could plug-and-play with?

Output:
The generalization and learning speed of a multi-class neural network can often be significantly improved by using soft targets that are a weighted average of the hard targets and the uniform distribution over labels. Smoothing the labels in this way prevents the network from becoming over-confident and label smoothing has been used in many state-of-the-art models, including image classification, language translation, and speech recognition.

Label Smoothing is already implemented in Tensorflow within the cross-entropy loss functions. BinaryCrossentropy, CategoricalCrossentropy. But currently, there is no official implementation of Label Smoothing in PyTorch. However, there is going an active discussion on it and hopefully, it will be provided with an official package. Here is that discussion thread: Issue #7455.
Here We will bring some available best implementation of Label Smoothing (LS) from PyTorch practitioner. Basically, there are many ways to implement the LS. Please refer to this specific discussion on this, one is here, and another here. Here we will bring implementation in 2 unique ways with two versions of each; so total 4.
Option 1: CrossEntropyLossWithProbs
In this way, it accepts the one-hot target vector. The user must manually smooth their target vector. And it can be done within with torch.no_grad() scope, as it temporarily sets all of the requires_grad flags to false.

Devin Yang: Source

import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn.modules.loss import _WeightedLoss

class LabelSmoothingLoss(nn.Module):
    def __init__(self, classes, smoothing=0.0, dim=-1, weight = None):
        """if smoothing == 0, it's one-hot method
           if 0 < smoothing < 1, it's smooth method
        """
        super(LabelSmoothingLoss, self).__init__()
        self.confidence = 1.0 - smoothing
        self.smoothing = smoothing
        self.weight = weight
        self.cls = classes
        self.dim = dim

    def forward(self, pred, target):
        assert 0 <= self.smoothing < 1
        pred = pred.log_softmax(dim=self.dim)

        if self.weight is not None:
            pred = pred * self.weight.unsqueeze(0)

        with torch.no_grad():
            true_dist = torch.zeros_like(pred)
            true_dist.fill_(self.smoothing / (self.cls - 1))
            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))

Additionally, we've added an assertion checkmark on self. smoothing and added loss weighting support on this implementation.

Shital Shah: Source

Shital already posted the answer here. Here we're pointing out that this implementation is similar to Devin Yang's above implementation. However, here we're mentioning his code with minimizing a bit of code syntax.
class SmoothCrossEntropyLoss(_WeightedLoss):
    def __init__(self, weight=None, reduction='mean', smoothing=0.0):
        super().__init__(weight=weight, reduction=reduction)
        self.smoothing = smoothing
        self.weight = weight
        self.reduction = reduction

    def k_one_hot(self, targets:torch.Tensor, n_classes:int, smoothing=0.0):
        with torch.no_grad():
            targets = torch.empty(size=(targets.size(0), n_classes),
                                  device=targets.device) \
                                  .fill_(smoothing /(n_classes-1)) \
                                  .scatter_(1, targets.data.unsqueeze(1), 1.-smoothing)
        return targets

    def reduce_loss(self, loss):
        return loss.mean() if self.reduction == 'mean' else loss.sum() \
        if self.reduction == 'sum' else loss

    def forward(self, inputs, targets):
        assert 0 <= self.smoothing < 1

        targets = self.k_one_hot(targets, inputs.size(-1), self.smoothing)
        log_preds = F.log_softmax(inputs, -1)

        if self.weight is not None:
            log_preds = log_preds * self.weight.unsqueeze(0)

        return self.reduce_loss(-(targets * log_preds).sum(dim=-1))

Check
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torch.nn.modules.loss import _WeightedLoss

if __name__=="__main__":
    # 1. Devin Yang
    crit = LabelSmoothingLoss(classes=5, smoothing=0.5)
    predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
                                 [0, 0.9, 0.2, 0.2, 1],
                                 [1, 0.2, 0.7, 0.9, 1]])
    v = crit(Variable(predict),
             Variable(torch.LongTensor([2, 1, 0])))
    print(v)

    # 2. Shital Shah
    crit = SmoothCrossEntropyLoss(smoothing=0.5)
    predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
                                 [0, 0.9, 0.2, 0.2, 1],
                                 [1, 0.2, 0.7, 0.9, 1]])
    v = crit(Variable(predict),
             Variable(torch.LongTensor([2, 1, 0])))
    print(v)

tensor(1.4178)
tensor(1.4178)

Option 2: LabelSmoothingCrossEntropyLoss
By this, it accepts the target vector and uses doesn't manually smooth the target vector, rather the built-in module takes care of the label smoothing. It allows us to implement label smoothing in terms of F.nll_loss.
(a). Wangleiofficial: Source - (AFAIK), Original Poster
(b). Datasaurus: Source  - Added Weighting Support
Further, we slightly minimize the coding write-up to make it more concise.
class LabelSmoothingLoss(torch.nn.Module):
    def __init__(self, smoothing: float = 0.1,
                 reduction="mean", weight=None):
        super(LabelSmoothingLoss, self).__init__()
        self.smoothing   = smoothing
        self.reduction = reduction
        self.weight    = weight

    def reduce_loss(self, loss):
        return loss.mean() if self.reduction == 'mean' else loss.sum() \
         if self.reduction == 'sum' else loss

    def linear_combination(self, x, y):
        return self.smoothing * x + (1 - self.smoothing) * y

    def forward(self, preds, target):
        assert 0 <= self.smoothing < 1

        if self.weight is not None:
            self.weight = self.weight.to(preds.device)

        n = preds.size(-1)
        log_preds = F.log_softmax(preds, dim=-1)
        loss = self.reduce_loss(-log_preds.sum(dim=-1))
        nll = F.nll_loss(
            log_preds, target, reduction=self.reduction, weight=self.weight
        )
        return self.linear_combination(loss / n, nll)

NVIDIA/DeepLearningExamples: Source

class LabelSmoothing(nn.Module):
    """NLL loss with label smoothing.
    """
    def __init__(self, smoothing=0.0):
        """Constructor for the LabelSmoothing module.
        :param smoothing: label smoothing factor
        """
        super(LabelSmoothing, self).__init__()
        self.confidence = 1.0 - smoothing
        self.smoothing = smoothing

    def forward(self, x, target):
        logprobs = torch.nn.functional.log_softmax(x, dim=-1)
        nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1))
        nll_loss = nll_loss.squeeze(1)
        smooth_loss = -logprobs.mean(dim=-1)
        loss = self.confidence * nll_loss + self.smoothing * smooth_loss
        return loss.mean()

Check
if __name__=="__main__":
    # Wangleiofficial
    crit = LabelSmoothingLoss(smoothing=0.3, reduction="mean")
    predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
                                 [0, 0.9, 0.2, 0.2, 1],
                                 [1, 0.2, 0.7, 0.9, 1]])

    v = crit(Variable(predict),
             Variable(torch.LongTensor([2, 1, 0])))
    print(v)

    # NVIDIA
    crit = LabelSmoothing(smoothing=0.3)
    predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1, 0],
                                 [0, 0.9, 0.2, 0.2, 1],
                                 [1, 0.2, 0.7, 0.9, 1]])
    v = crit(Variable(predict),
             Variable(torch.LongTensor([2, 1, 0])))
    print(v)

tensor(1.3883)
tensor(1.3883)

Update: Officially Added
torch.nn.CrossEntropyLoss(weight=None, size_average=None,
                          ignore_index=- 100, reduce=None,
                          reduction='mean', label_smoothing=0.0)


Instruction: How downsample work in ResNet in pytorch code?

Input:
In this pytorch ResNet code example they define downsample as variable in line 44. and line 58 use it as function. How this downsample work here as CNN point of view and as python Code point of view.

code example : pytorch ResNet

i searched for if downsample is any pytorch inbuilt function. but it is not.

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, norm_layer=None):
        super(BasicBlock, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        if groups != 1:
            raise ValueError('BasicBlock only supports groups=1')
        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = norm_layer(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = norm_layer(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

return out

Output:
In this ResNet example, Here when we define BasicBlock class we pass downsample as constructor parameter.

def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, norm_layer=None):

if we pass nothing to class then downsample = None , as result identity will not changed.

When we pass downsample = "some convolution layer" as class constructor argument, It will downsample the identity via passed convolution layer to sucessfully perform addition. this layer will downsample the identity through code as mentioned

  if self.downsample is not None:
        identity = self.downsample(x)


Instruction: Why does dim=1 return row indices in torch.argmax?

Input:
I am working on argmax function of PyTorch which is defined as:

torch.argmax(input, dim=None, keepdim=False)

Consider an example

a = torch.randn(4, 4)
print(a)
print(torch.argmax(a, dim=1))

Here when I use dim=1 instead of searching column vectors, the function searches for row vectors as shown below.

print(a) :
tensor([[-1.7739,  0.8073,  0.0472, -0.4084],
        [ 0.6378,  0.6575, -1.2970, -0.0625],
        [ 1.7970, -1.3463,  0.9011, -0.8704],
        [ 1.5639,  0.7123,  0.0385,  1.8410]])

print(torch.argmax(a, dim=1))
tensor([1, 1, 0, 3])

As far as my assumption goes dim = 0 represents rows and dim =1 represent columns.

Output:
It's time to correctly understand how the axis or dim argument work in PyTorch:

The following example should make sense once you comprehend the above picture:

    |
    v
  dim-0  ---> -----> dim-1 ------> -----> --------> dim-1
    |   [[-1.7739,  0.8073,  0.0472, -0.4084],
    v    [ 0.6378,  0.6575, -1.2970, -0.0625],
    |    [ 1.7970, -1.3463,  0.9011, -0.8704],
    v    [ 1.5639,  0.7123,  0.0385,  1.8410]]
    |
    v

# argmax (indices where max values are present) along dimension-1
In [215]: torch.argmax(a, dim=1)
Out[215]: tensor([1, 1, 0, 3])

Note: dim (short for 'dimension') is the torch equivalent of 'axis' in NumPy.


Instruction: How to create a Pytorch Dataset from .pt files?

Input:
I have transformed MNIST images saved as .pt files in a folder in Google drive. I'm writing my Pytorch code in Colab.

I would like to use these files, and create a Dataset that stores these images as Tensors. How can I do this?

Transforming images during training took too long. Hence, transformed them and saved them all as .pt files. I just want to load them back as a dataset and use them in my model.

Output:
The approach you are following to save images is indeed a good idea. In such a case, you can simply write your own Dataset class to load the images.

from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import RandomSampler

class ReaderDataset(Dataset):
    def __init__(self, filename):
        # load the images from file

    def __len__(self):
        # return total dataset size

    def __getitem__(self, index):
        # write your code to return each batch element

Then you can create Dataloader as follows.

train_dataset = ReaderDataset(filepath)
train_sampler = RandomSampler(train_dataset)
train_loader = DataLoader(
    train_dataset,
    batch_size=args.batch_size,
    sampler=train_sampler,
    num_workers=args.data_workers,
    collate_fn=batchify,
    pin_memory=args.cuda,
    drop_last=args.parallel
)
# args is a dictionary containing parameters
# batchify is a custom function that prepares each mini-batch


Instruction: How to fix "TypeError: data type not understood" in numpy when creating transformer no peak mask

Input:
I'm trying to implement and train a transformer for NMT via a blog post, everything works except I can't create the no peaking mask as I get this error: "TypeError: data type not understood"

Code:

target_seq = batch.Python.transpose(0,1)
target_pad = PY_TEXT.vocab.stoi['<pad>']
target_msk = (target_seq != target_pad).unsqueeze(1)
size = target_seq.size(1) # get seq_len for matrix
nopeak_mask = np.triu(np.ones(1, size, size),
k=1).astype('uint8')
nopeak_mask = Variable(torch.from_numpy(nopeak_mask) == 0)
target_msk = target_msk & nopeak_mask

Error message:

TypeError                                 Traceback (most recent call last)
<ipython-input-36-e19167b74ba0> in <module>()
      4 target_msk = (target_seq != target_pad).unsqueeze(1)
      5 size = target_seq.size(1) # get seq_len for matrix
----> 6 nopeak_mask = np.triu(np.ones(1, size, size),
      7 k=1).astype('uint8')
      8 nopeak_mask = Variable(torch.from_numpy(nopeak_mask) == 0)

~/.local/lib/python3.6/site-packages/numpy/core/numeric.py in ones(shape, dtype, order)
    201
    202     """
--> 203     a = empty(shape, dtype, order)
    204     multiarray.copyto(a, 1, casting='unsafe')
    205     return a

TypeError: data type not understood

Output:
The first input to np.triu should be a tuple of desired sizes instead of a numpy array.

Try:

np.triu((1, size, size), k=1).astype("uint8")


Instruction: How can I simplify a nested loop into torch tensor operations?

Input:
I'm trying to convert some code I have written in numpy which contains a nested-loop into tensor operations found in PyTorch. However, after trying to implement my own version I'm not getting the same value on the output. I have managed to do the same with a single loop, so I'm not entirely sure what I'm doing wrong.

    #(Numpy Version)
    #calculate Kinetic Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        summation += (k_values[i]**2.0)*wavefp[i]*(((self.hbar*kp_values[i])**2.0)/(2.0*self.mu))*wavef[i]
    Ek = step*(4.0*np.pi)*summation

    #(Numpy Version)
    #calculate Potential Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        for j in range(0,len(kp_values)-1):
            summation+= (k_values[i]**2.0)*wavefp[i]*(kp_values[j]**2.0)*wavef[j]*self.MTV[i,j]
    Ep = (step**2.0)*(4.0*np.pi)*(2.0/np.pi)*summation

            #####################################################

    #(PyTorch Version)
    #calcualte Kinetic Energy
    Ek = step*(4.0*np.pi)*torch.sum( k_values.pow(2)*wavefp.mul(wavef)*((kp_values.mul(self.hbar)).pow(2)/(2.0*self.mu)) )

    #(PyTorch Version)
    #calculate Potential Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        summation += ((k_values[i].pow(2)).mul(wavefp[i]))*torch.sum( (kp_values.pow(2)).mul(wavef).mul(self.MTV[i,:]) )
    Ep = (step**2.0)*(4.0*np.pi)*(2.0/np.pi)*summation

The arrays/tensors k_values, kp_values, wavef, and wavefp have dimensions of (1000,1). The values self.hbar, and self.mu, and step are scalars. The variable self.MTV is a matrix of size (1000,1000).

I would expect that both methods would give the same output but they don't. The code for calculating the Kinetic Energy (in both Numpy and PyTorch) give the same value. However, the potential energy calculation differ, and I'm not entirely sure why.

Many Thanks in advance!

Output:
The problem is in the shapes. You have kp_values and wavef in (1000, 1) which needs to be converted to (1000, ) before the multiplications. The outcome of (kp_values.pow(2)).mul(wavef).mul(MTV[i,:]) is a matrix but you asummed it is a vector.

So, the following should work.

summation += ((k_values[i].pow(2)).mul(wavefp[i]))*torch.sum((kp_values.squeeze(1)
    .pow(2)).mul(wavef.squeeze(1)).mul(MTV[i,:]))

And a loop-free Numpy and PyTorch solution would be:

step = 1.0
k_values = np.random.randint(0, 100, size=(1000, 1)).astype("float") / 100
kp_values = np.random.randint(0, 100, size=(1000, 1)).astype("float") / 100
wavef = np.random.randint(0, 100, size=(1000, 1)).astype("float") / 100
wavefp = np.random.randint(0, 100, size=(1000, 1)).astype("float") / 100
MTV = np.random.randint(0, 100, size=(1000, 1000)).astype("float") / 100

# Numpy solution
term1 = k_values**2.0 * wavefp # 1000 x 1
temp = kp_values**2.0 * wavef # 1000 x 1
term2 = np.matmul(temp.transpose(1, 0), MTV).transpose(1, 0) # 1000 x 1000
summation = np.sum(term1 * term2)
print(summation)

# PyTorch solution
term1 = k_values.pow(2).mul(wavefp) # 1000 x 1
term2 = kp_values.pow(2).mul(wavef).transpose(0, 1).matmul(MTV) # 1000 x 1000
summation = torch.sum(term2.transpose(0, 1).mul(term1)) # 1000 x 1000
print(summation.item())

Output

12660.407492918514
12660.407492918514


Instruction: What do * and mean stand for in this PyTorch expression?

Input:
I do not understand how to evaluate this expression:
x.view(*(x.shape[:-2]),-1).mean(-1)`,

if x.shape == (N, C, H, W).
What does the asterisk * stand for? And what is mean(-1) ?

Output:
What is *?

For .view() pytorch expects the new shape to be provided by individual int arguments (represented in the doc as *shape). The asterisk (*) can be used in python to unpack a list into its individual elements, thus passing to view the correct form of input arguments it expects.
So, in your case, x.shape is (N, C, H, W), if you were to pass x.shape[:-2] without the asterisk, you would get x.view((N, C), -1) - which is not what view() expects. Unpacking (N, C) using the asterisk results with view receiving view(N, C, -1) arguments as it expects. The resulting shape is (N, C, H*W) (a 3D tensor instead of 4).

What is mean(-1)?

Simply look at the documentation of .mean(): the first argument is a dim argument. That is x.mean(-1) applies mean along the last dimension. In your case, since keepdim=False by default, your output will be a (N, C) sized tensor where each element correspond to the mean value along both spatial dimensions.
This is equivalent to
x.mean(-1).mean(-1)


Instruction: Creating a Simple 1D CNN in PyTorch with Multiple Channels

Input:
The dimensionality of the PyTorch inputs are not what the model expects, and I am not sure why.

To my understanding...

in_channels is first the number of 1D inputs we would like to pass to the model, and is the previous out_channel for all subsequent layers.

out_channels is the desired number of kernels (filters).

kernel_size is the number of parameters per filter.

Therefore, we would expect, as data passed to forward, a dataset with 7 1D channels (i.e. a 2D input).

However, the following code throws an error that is not consistent with what I expect, where this code:

import numpy
import torch

X = numpy.random.uniform(-10, 10, 70).reshape(-1, 7)
# Y = np.random.randint(0, 9, 10).reshape(-1, 1)

class Simple1DCNN(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN, self).__init__()
        self.layer1 = torch.nn.Conv1d(in_channels=7, out_channels=20, kernel_size=5, stride=2)
        self.act1 = torch.nn.ReLU()
        self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
    def forward(self, x):
        x = self.layer1(x)
        x = self.act1(x)
        x = self.layer2(x)

        log_probs = torch.nn.functional.log_softmax(x, dim=1)

        return log_probs

model = Simple1DCNN()
print(model(torch.tensor(X)).size)

Throws the following error:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-5-eca5856a2314> in <module>()
     21
     22 model = Simple1DCNN()
---> 23 print(model(torch.tensor(X)).size)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-5-eca5856a2314> in forward(self, x)
     12         self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
     13     def forward(self, x):
---> 14         x = self.layer1(x)
     15         x = self.act1(x)
     16         x = self.layer2(x)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    185     def forward(self, input):
    186         return F.conv1d(input, self.weight, self.bias, self.stride,
--> 187                         self.padding, self.dilation, self.groups)
    188
    189

RuntimeError: Expected 3-dimensional input for 3-dimensional weight [20, 7, 5], but got 2-dimensional input of size [10, 7] instead

Edit: See below for solution, motivated by Shai.

import numpy
import torch

X = numpy.random.uniform(-10, 10, 70).reshape(1, 7, -1)
# Y = np.random.randint(0, 9, 10).reshape(1, 1, -1)

class Simple1DCNN(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN, self).__init__()
        self.layer1 = torch.nn.Conv1d(in_channels=7, out_channels=20, kernel_size=5, stride=2)
        self.act1 = torch.nn.ReLU()
        self.layer2 = torch.nn.Conv1d(in_channels=20, out_channels=10, kernel_size=1)
    def forward(self, x):
        x = self.layer1(x)
        x = self.act1(x)
        x = self.layer2(x)

        log_probs = torch.nn.functional.log_softmax(x, dim=1)

        return log_probs

model = Simple1DCNN().double()
print(model(torch.tensor(X)).shape)

Output:
You are forgetting the "minibatch dimension", each "1D" sample has indeed two dimensions: the number of channels (7 in your example) and length (10 in your case). However, pytorch expects as input not a single sample, but rather a minibatch of B samples stacked together along the "minibatch dimension".
So a "1D" CNN in pytorch expects a 3D tensor as input: BxCxT. If you only have one signal, you can add a singleton dimension:

 out = model(torch.tensor(X)[None, ...])


Instruction: TypeError: can't convert np.ndarray of type numpy.object_

Input:
How to convert a numpy array of dtype=object to torch Tensor?
array([
   array([0.5, 1.0, 2.0], dtype=float16),
   array([4.0, 6.0, 8.0], dtype=float16)
], dtype=object)

Output:
It is difficult to answer properly since you do not show us how you try to do it. From your error message I can see that you try to convert a numpy array containing objects to a torch tensor. This does not work, you will need a numeric data type:
import torch
import numpy as np

# Your test array without 'dtype=object'
a = np.array([
    np.array([0.5, 1.0, 2.0], dtype=np.float16),
    np.array([4.0, 6.0, 8.0], dtype=np.float16),
])

b = torch.from_numpy(a)

print(a.dtype) # This should not be 'object'
print(b)

Output
float16
tensor([[0.5000, 1.0000, 2.0000],
        [4.0000, 6.0000, 8.0000]], dtype=torch.float16)


Instruction: Pytorch: multiple datasets with multiple losses

Input: I am using multiple datasets. I have multiple losses, each of which must be evaluated on a subset of these datasets. I want to generate a batch from each dataset, and evaluate each loss on all of its appropriate batches. Some of the losses are pairwise (need to load pairs of corresponding datapoints) whereas others are computed on single datapoints. I need to design this in such a way that is open to easily adding new datasets. Is there any pytorch builtin that would help with this? What is the best way to design this in pytorch? Thanks in advance.

Output:
It's not clear from your question what exactly your settings are.
However, you can have multiple Datasets instances, one for each of your datasets.
On top of your datasets, you can implement a "tagged dataset", a dataset that adds a "tag" for all samples:

class TaggedDataset(data.Dataset):
  def __init__(dataset, tag):
    super(TaggedDataset, self).__init__()
    self.ds_ = dataset
    self.tag_ = tag

  def __len__(self):
    return len(self.ds_)

  def __getitem__(self, index):
    return self.ds_[index], self.tag_

Give a different tag to each dataset, concat all of them into a single ConcatDataset, and wrap a regular DataLoader around it.

Now, in your training code

for input, label, tag in my_tagged_loader:
  # process each input according to the dataset tag it got.


Instruction: Which python deep learning libraries compile at runtime?

Input:
I am trying to wrap my head around C-optimized code in python. I have read a couple of times now that python achieves high-speed computing through C-extensions. In other words, whenever I work with libraries such as numpy, it basically calls a C-extension that calculates the result and returns it.

C-extensions using numpy

Say I want to add two numbers using np.add(x,y). If I understand it correctly, libraries such as numpy do not compile the python code but instead already come with executables that will simply take the values x and y and return the result. Is that correct?

Theano, Tensorflow, and PyTorch

In particular, I am wondering if this is also true for deep learning libraries. According to the official documentation of Theano, it requires g++ and gcc (at least they are highly recommended). Does this mean that Theano will compile C (or C++) code at runtime of the python script? If so, is it the same for PyTorch and Tensorflow?

I hope that someone can solve my confusion here! Thanks a lot!

Output:
C extensions in python

numpy uses C-extensions a lot. For instance, you can take a look at the C implementation of the sort() function [1] here [2].

[1] https://docs.scipy.org/doc/numpy/reference/generated/numpy.sort.html

[2] https://github.com/numpy/numpy/blob/master/numpy/core/src/npysort/quicksort.c.src

Deep learning libraries

Deep learning libraries use C-extensions for a large part of their backend, as well as CUDA and CUDNN. Code can be compiled at runtime:

[3] http://deeplearning.net/software/theano/extending/pipeline.html#compilation-of-the-computation-graph

[4] https://www.tensorflow.org/xla/jit

[5] https://pytorch.org/blog/the-road-to-1_0/#production--pain-for-researchers

To answer your question, theano will compile C/C++ code at runtime of the python script. The graph compilation time at runtime is extremely slow for theano: I advise you to focus on pytorch or tensorflow rather than theano.

If you're new to deep learning, you may take a quick look at [6] too.

[6] https://github.com/google/jax


Instruction: Stretch the values of a pytorch tensor

Input:
I wish to 'stretch' the last two dimensions of a pytorch tensor to increase the spatial resolution of a (batch, channels, y, x) tensor.

Minimal example (I need 'new_function')

a = torch.tensor([[1, 2], [3, 4]])
b = new_function(a, (2, 3))
print(b)
tensor([[1, 1, 1, 2, 2, 2],
        [1, 1, 1, 2, 2, 2],
        [3, 3, 3, 4, 4, 4],
        [3, 3, 3, 4, 4, 4]])

One way of doing this (for the real problem):

a = torch.ones((2, 256, 2, 2)) # my original data.
b = torch.zeros((2, 256, 80, 96)) # The output I need
b[:, :, :40, :48] = a[:, :, 0, 0]
b[:, :, 40:, :48] = a[:, :, 1, 0]
b[:, :, :40, 48:] = a[:, :, 0, 1]
b[:, :, 40:, 48:] = a[:, :, 1, 1]

Output:
Use torch.nn.functional.interpolate (thanks to Shai)

torch.nn.functional.interpolate(input_tensor.float(), size=(4, 6))

My original idea was to use a variety of view and repeat methods:

def stretch(e, sdims):
    od = e.shape
    return e.view(od[0], od[1], -1, 1).repeat(1, 1, 1, sdims[-1]).view(od[0], od[1], od[2], -1).repeat(1, 1, 1, sdims[-2]).view(od[0], od[1], od[2] * sdims[0], od[3] * sdims[1])

torch.Size([2, 2, 4, 6])


Instruction: What is the desired behavior of average pooling with padding?

Input:
Recently I've trained a neural network using pytorch and there is an average pooling layer with padding in it. And I'm confused about the behavior of it as well as the definition of average pooling with padding.

For example, if we have a input tensor:

[[1, 2, 3],
 [4, 5, 6],
 [7, 8, 9]]

When padding is one and kernel size 3, the input to the first kernel should be:

 0, 0, 0
 0, 1, 2
 0, 4, 5

The output from the pytorch is 12/4 = 3 (ignoring padded 0)， but I think it should be 12/9 = 1.333

Can anyone explain this to me?

Much appreciated.

Output:
It's basically up to you to decide how you want your padded pooling layer to behave.
This is why pytorch's avg pool (e.g., nn.AvgPool2d) has an optional parameter count_include_pad=True:
By default (True) Avg pool will first pad the input and then treat all elements the same. In this case the output of your example would indeed be 1.33.
On the other hand, if you set count_include_pad=False the pooling layer will ignore the padded elements and the result in your example would be 3.


Instruction: Getting gradient of vectorized function in pytorch

Input:
I am brand new to PyTorch and want to do what I assume is a very simple thing but am having a lot of difficulty.

I have the function sin(x) * cos(x) + x^2 and I want to get the derivative of that function at any point.

If I do this with one point it works perfectly as

x = torch.autograd.Variable(torch.Tensor([4]),requires_grad=True)
y = torch.sin(x)*torch.cos(x)+torch.pow(x,2)
y.backward()
print(x.grad) # outputs tensor([7.8545])

However, I want to be able to pass in a vector as x and for it to evaluate the derivative element-wise. For example:

Input: [4., 4., 4.,]
Output: tensor([7.8545, 7.8545, 7.8545])

But I can't seem to get this working.

I tried simply doing

x = torch.tensor([4., 4., 4., 4.], requires_grad=True)
out = torch.sin(x)*torch.cos(x)+x.pow(2)
out.backward()
print(x.grad)

But I get the error "RuntimeError: grad can be implicitly created only for scalar outputs"

How do I adjust this code for vectors?

Thanks in advance,

Output:
Here you can find relevant discussion about your error.

In essence, when you call backward() without arguments it is implicitly converted to backward(torch.Tensor([1])), where torch.Tensor([1]) is the output value with respect to which gradients are calculated.

If you pass 4 (or more) inputs, each needs a value with respect to which you calculate gradient. You can pass torch.ones_like explicitly to backward like this:

import torch

x = torch.tensor([4.0, 2.0, 1.5, 0.5], requires_grad=True)
out = torch.sin(x) * torch.cos(x) + x.pow(2)
# Pass tensor of ones, each for each item in x
out.backward(torch.ones_like(x))
print(x.grad)


Instruction: Extract elements from .npy file, convert them to PyTorch tensors

Input:
I read a .npy file that contains just the labels for images. The labels are stored in dictionary format. I need to convert this to an array of Tensors. But I'm unable to extract elements one of the other from the object the file returns, which is numpy.ndarray type.

import numpy as np
data = np.load('/content/drive/My Drive/targets.npy')
print(data.item())

{0: array(5), 1: array(0), 2: array(4), 3: array(1), 4: array(9), 5: array(2), 6: array(1), 7: array(3)}

print(data[()].values())

dict_values([array(5), array(0), array(4), array(1), array(9), array(2), array(1), array(3)])

I would like to create an array of tensors instead.

Thanks in advance.

Output:
The below worked for me, with guidance by @kmario23

import numpy as np
data = np.load('/content/drive/My Drive/targets.npy')
print(data.item())

{0: array(5), 1: array(0), 2: array(4), 3: array(1), 4: array(9), 5: array(2), 6: array(1), 7: array(3)}
# data is a 0-d numpy.ndarray that contains a dictionary.

print(list(data[()].values()))

[array(5),
 array(0),
 array(4),
 array(1),
 array(9),
 array(2),
 array(1),
 array(3),
 array(1),
 array(4),
 array(3)]

# torch.Tensor(5) gives tensor([2.0581e-35, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00])
# torch.tensor(5) gives 5
# unsure of why the difference exists..

Labels = torch.stack([torch.tensor(i) for i in list_of_labels_array_form])

print(Labels)

tensor([5, 0, 4,  ..., 2, 5, 0])


Instruction: Replicate subtensors in PyTorch

Input:
I have a tensor “image_features” having shape torch.Size([100, 1024, 14, 14]). I need to replicate each subtensor (1024, 14, 14) 10 times, obtaining a tensor having shape torch.Size([1000, 1024, 14, 14]).

Basically, the first ten rows of the resulting tensor should correspond to the first row of the original one, the following ten rows of the resulting tensor should correspond to the second row of the original one, and so on. If possible, I don’t want to create a copy (each replicated subtensor can share the memory with the tensor it is replicated from), but it is ok to create a copy if there isn’t any other way.

How can I do it?

Thank you very much.

Output:
Another approach that would solve your problem is:

orig_shape = (100, 1024, 14, 14)
new_shape = (100, 10, 1024, 14, 14)
input = torch.randn(orig_shape) # [100, 1024, 14, 14]
input = input.unsqueeze(1) # [100, 1, 1024, 14, 14]
input = input.expand(*new_shape) # [100, 10, 1024, 14, 14]
input = input.transpose(0, 1).contiguous() # [10, 100, 1024, 14, 14]
input = input.view(-1, *orig_shape[1:]) # [1000, 1024, 14, 14]

We can verify it.

orig_shape = (2, 3, 4)
new_shape = (2, 5, 3, 4)
input = torch.randn(orig_shape)
print(input)
input = input.unsqueeze(1)
input = input.expand(*new_shape)
input = input.transpose(0, 1).contiguous()
input = input.view(-1, *orig_shape[1:])
print(input)

The code snippet results in:

tensor([[[-1.1728,  1.0421, -1.0716,  0.6456],
     [-1.2214,  1.1484, -0.1436,  1.2353],
     [-0.4395, -0.9473, -0.1382, -0.9357]],

    [[-0.4735, -1.4329, -0.0025, -0.6384],
     [ 0.5102,  0.7813,  1.2810, -0.6013],
     [ 0.6152,  1.1734, -0.4591, -1.7447]]])

tensor([[[-1.1728,  1.0421, -1.0716,  0.6456],
     [-1.2214,  1.1484, -0.1436,  1.2353],
     [-0.4395, -0.9473, -0.1382, -0.9357]],

    [[-0.4735, -1.4329, -0.0025, -0.6384],
     [ 0.5102,  0.7813,  1.2810, -0.6013],
     [ 0.6152,  1.1734, -0.4591, -1.7447]],

    [[-1.1728,  1.0421, -1.0716,  0.6456],
     [-1.2214,  1.1484, -0.1436,  1.2353],
     [-0.4395, -0.9473, -0.1382, -0.9357]],

    [[-0.4735, -1.4329, -0.0025, -0.6384],
     [ 0.5102,  0.7813,  1.2810, -0.6013],
     [ 0.6152,  1.1734, -0.4591, -1.7447]]])


Instruction: RuntimeError: cuda runtime error (35) : CUDA driver version is insufficient for CUDA runtime version at torch/csrc/cuda/Module.cpp:51

Input:
When I try to load a pytorch checkpoint:

checkpoint = torch.load(pathname)

I see:

  RuntimeError: cuda runtime error (35) : CUDA driver version is insufficient for CUDA runtime version at torch/csrc/cuda/Module.cpp:51

I created the checkpoint with a GPU available, but now only have CPU available.

How do I load the checkpoint?

Output:
Load the checkpoint data to the best currently available location:

if torch.cuda.is_available():
    map_location=lambda storage, loc: storage.cuda()
else:
    map_location='cpu'

checkpoint = torch.load(pathname, map_location=map_location)


Instruction: Expected object of scalar type Long but got scalar type Byte for argument #2 'target'

Input:
I am running a nn on colab and came across this error which was not there when i ran the same code on my local system. I have tried with reduced batch size too but the error still persists.

Loading dataset
Start training
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-37432f9d142a> in <module>()
     70           start_epoch=start_epoch, log=log_interval,
     71           checkpoint_path=os.path.join(dataset_dir, "cnn_block_frame_flow"),
---> 72           validate=True, resume=False, flow=True, use_cuda=cuda)
     73
     74     #model = models.model()

/content/KTH-Action-Recognition/main/train_helper.py in train(model, num_epochs, train_set, dev_set, lr, batch_size, start_epoch, log, checkpoint_path, validate, resume, flow, use_cuda)
    107             outputs = get_outputs(model, samples["instance"], flow=flow,
    108                                   use_cuda=use_cuda)
--> 109             loss = criterion(outputs, labels)
    110             loss.backward()
    111             optimizer.step()

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)
    902     def forward(self, input, target):
    903         return F.cross_entropy(input, target, weight=self.weight,
--> 904                                ignore_index=self.ignore_index, reduction=self.reduction)
    905
    906

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   1968     if size_average is not None or reduce is not None:
   1969         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 1970     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   1971
   1972

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1788                          .format(input.size(0), target.size(0)))
   1789     if dim == 2:
-> 1790         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   1791     elif dim == 4:
   1792         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: Expected object of scalar type Long but got scalar type Byte for argument #2 'target'

Can someone tell me what is causing this error? thank you

Output:
The title of your question is telling what is causing this error. The target should have type torch.LongTensor, but it is instead torch.ByteTensor. Before calling nll_loss do:

target = target.type(torch.LongTensor)


Instruction: How to load pretrained googlenet model in pytorch

Input:
I'm trying to finetune a GoogleNet network over a specific dataset but I'm having trouble loading it. What I try now is:

model = torchvision.models.googlenet(pretrained=True)

However I get an error:

AttributeError: module 'torchvision.models' has no attribute 'googlenet'

I have the latest version of torchvision but reinstalled just to be sure, the error is still there.

Output:
You can instead use the GoogLeNet inception_v3 model ("Rethinking the Inception Architecture for Computer Vision"):

import torchvision
google_net = torchvision.models.inception_v3(pretrained=True)


Instruction: Got Very Different Scores After Translating Simple Test Model from Keras to PyTorch

Input:
I'm trying to transition from Keras to PYTorch.

After reading tutorials and similar questions, I came up with the following simple models to test. However, the two models below gives me very different scores: Keras (0.9), PyTorch (0.03).

Could someone give me guidance?

Basically my dataset has 120 features and multilabels with 3 classes that look like below.

[
    [1,1,1],
    [0,1,1],
    [1,0,0],
    ...
]

def score(true, pred):
    lrl = label_ranking_loss(true, pred)
    lrap = label_ranking_average_precision_score(true, pred)
    print('LRL:', round(lrl), 'LRAP:', round(lrap))

#Keras:
model= Sequential()
model.add(Dense(60, activation="relu", input_shape=(120,)))
model.add(Dense(30, activation="relu"))
model.add(Dense(3, activation="sigmoid"))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=32, epochs=100)
pred = model.predict(x_test)
score(y_test, pred)

#PyTorch
model = torch.nn.Sequential(
    torch.nn.Linear(120, 60),
    torch.nn.ReLU(),
    torch.nn.Linear(60, 30),
    torch.nn.ReLU(),
    torch.nn.Linear(30, 3),
    torch.nn. Sigmoid())
loss_fn = torch.nn. BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

epochs = 100
batch_size = 32
n_batch = int(x_train.shape[0]/batch_size)
for epoch in range(epochs):
    avg_cost = 0
    for i in range(n_batch):
        x_batch = x_train[i*batch_size:(i+1)*batch_size]
        y_batch = y_train[i*batch_size:(i+1)*batch_size]
        x, y = Variable(torch.from_numpy(x_batch).float()), Variable(torch.from_numpy(y_batch).float(), requires_grad=False)
        pred = model(x)
        loss = loss_fn(pred, y)
        loss.backward()
        optimizer.step()
        avg_cost += loss.item()/n_batch
    print(epoch, avg_cost)

x, y = Variable(torch.from_numpy(x_test).float()), Variable(torch.from_numpy(y_test).float(), requires_grad=False)
pred = model(x)
score(y_test, pred.data.numpy())

Output: You need to call optimizer.zero_grad() at the start of each iteration, otherwise the gradients from different batches just keep getting accumulated.


Instruction: Pytorch custom activation functions?

Input: I'm having issues with implementing custom activation functions in Pytorch, such as Swish. How should I go about implementing and using custom activation functions in Pytorch?

Output:
There are four possibilities depending on what you are looking for. You will need to ask yourself two questions:
Q1) Will your activation function have learnable parameters?
If yes, you have no choice but to create your activation function as an nn.Module class because you need to store those weights.
If no, you are free to simply create a normal function, or a class, depending on what is convenient for you.
Q2) Can your activation function be expressed as a combination of existing PyTorch functions?
If yes, you can simply write it as a combination of existing PyTorch function and won't need to create a backward function which defines the gradient.
If no you will need to write the gradient by hand.
Example 1: SiLU function
The SiLU function f(x) = x * sigmoid(x) does not have any learned weights and can be written entirely with existing PyTorch functions, thus you can simply define it as a function:
def silu(x):
    return x * torch.sigmoid(x)

and then simply use it as you would have torch.relu or any other activation function.
Example 2: SiLU with learned slope
In this case you have one learned parameter, the slope, thus you need to make a class of it.
class LearnedSiLU(nn.Module):
    def __init__(self, slope = 1):
        super().__init__()
        self.slope = slope * torch.nn.Parameter(torch.ones(1))

    def forward(self, x):
        return self.slope * x * torch.sigmoid(x)

Example 3: with backward
If you have something for which you need to create your own gradient function, you can look at this example: Pytorch: define custom function


Instruction: Neural networks fails to approximate simple multiplication and division

Input:
I am trying to fit simple feedforward neural networks on simple data where my goal is to just approximate (abc)/d

max_a=2
max_b = 3000
max_c=10
max_d=1

def generate_data(no_elements=10000):
    a = np.random.uniform(0,max_a,no_elements)
    b = np.random.uniform(1,max_b,no_elements)
    c=np.random.uniform(0.001,max_c,no_elements)
    d=np.random.uniform(0.00001,max_d,no_elements)
    df=pd.DataFrame({"a":a,"b":b,"c":c,"d":d})

    e=(df.a*df.b*df.c)/df.d
    df["e"]=e
    return(df)

this is how i am generating data

then  I did data normalization

df = generate_data(5000)
np_df=df.iloc[:,:4]
means=np.mean(np_df,axis=0,keepdims=True)
stds=np.std(np_df,axis=0,keepdims=True)
x_train = (np_df-means)/stds
y_train = np_df[:,4]

and I have  built a simple pytorch network for regression so as it has to predict 'e'

class network_Regression(nn.Module):
    def __init__(self,layers):
        super(network_Regression, self).__init__()
        self.linear = nn.ModuleList()
        self.relu = nn.ModuleList()
        self.layers = layers
        for i in range(len(layers)-1):
            self.linear.append(nn.Linear(layers[i],layers[i+1]))
            if i+1 !=  len(layers)-1:
                self.relu.append(nn.ReLU())

    def forward(self,out):
        for i in range(len(self.relu)):
            out = self.linear[i](out)
            out = self.relu[i](out)
        out = self.linear[-1](out)
        return out

model = network_Regression([4,10,10,1])
criterion= nn.MSELoss()
optimizer=optim.Adam(model.parameters())

but when I tried to train these networks tried epochs from [1000 to 0.5M]

still, it isn't able to find a simple formula ((abc)/d)=e

I tried to change various hidden layer levels but loss been around 9 digits

model.train()
num_epochs = 1000
loss_list=[]
for epoch in range(num_epochs):
    for batch_idx, (data, target) in enumerate(data_loader):
    #print(batch_idx)
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = model(data.float())
        loss = criterion(output, target.float())
        #print(batch_idx, loss.data[0])
        loss.backward()
        optimizer.step()
        if epoch >2:
            if batch_idx % 200 == 0:
                loss_list.append(loss.data.item())
        if batch_idx % 400 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
            epoch, batch_idx * len(data), len(data_loader.dataset),
            100. * batch_idx / len(data_loader), loss.data.item()))

Output:
Looks like neural networks are bad in multiplication and division.

check out this for details.

So basically I have to log transform my data, in the above case to approximate ((abc)/d)=e neural network has to figure out simple addition and subtraction.As per this question complicated multiplication and division becomes ln(a) + ln(b)+ln(c)-ln(d) =ln(e) and just after take inverse log of ln(e) this idea works well.