Instruction: My PyTorch Conv1d with an identity kernel does not produce the same output as the input

Input:
I have:

        print('\ninp', inp.min(), inp.mean(), inp.max())
        print(inp)
        out = self.conv1(inp)
        print('\nout1', out.min(), out.mean(), out.max())
        print(out)
        quit()

My min, mean and max for my inp is: inp tensor(9.0060e-05) tensor(0.1357) tensor(2.4454)

For my output, I have: out1 tensor(4.8751, grad_fn=<MinBackward1>) tensor(21.8416, grad_fn=<MeanBackward0>) tensor(54.9332, grad_fn=<MaxBackward1>)

My self.conv1 is:

        self.conv1 = torch.nn.Conv1d(
            in_channels=161,
            out_channels=161,
            kernel_size=11,
            stride=1,
            padding=5)
        self.conv1.weight.data = torch.zeros(self.conv1.weight.data.size())
        self.conv1.weight.data[:, :, 5] = 1.0
        self.conv1.bias.data = torch.zeros(self.conv1.bias.data.size())

So my weights look like: tensor([0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.])

So if I understand how convolution works, this should produce the same output. But it doesn't.

What am I doing wrong?

Output:
Always try to provide a Minimal, Reproducible Example.

It shouldn't. You are probably forgetting the summation. As stated in the docs:

  In the simplest case, the output value of the layer with input size
  (N, C_in, L) and output (N, C_out, L_out) can be precisely
  described as:

  where ⋆ is the valid cross-correlation operator, N is a batch size, C denotes a number of channels, L is a length of signal sequence.

Notice that, in your example, the mean after the conv (i.e., 21.8416) is approx. 161 times the mean before (i.e., 161 * 0.1357), and this is not a coincidence. Notice the same happening in the code below:

import torch
torch.manual_seed(0)

# define the fake input data
x = torch.rand(1, 3, 5)
# >>> x
# tensor([[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074],
#          [0.6341, 0.4901, 0.8964, 0.4556, 0.6323],
#          [0.3489, 0.4017, 0.0223, 0.1689, 0.2939]]])

# define the conv
conv1 = torch.nn.Conv1d(3, 3, kernel_size=5, stride=1, padding=2)
conv1.weight.data = torch.zeros(conv1.weight.data.size())
conv1.weight.data[:, :, 2] = 1.0
conv1.bias.data = torch.zeros(conv1.bias.data.size())

# print mean before
print(x.mean())
# tensor(0.4091)

# print mean after
print(conv1(x).mean())
# tensor(1.2273, grad_fn=<MeanBackward0>)

See? After the conv, the mean is 3 times the original one.

As @jodag said, if you want an identity, you can do like this:

import torch
torch.manual_seed(0)

# define the fake input data
x = torch.rand(1, 3, 5)
# >>> x
# tensor([[[0.4963, 0.7682, 0.0885, 0.1320, 0.3074],
#          [0.6341, 0.4901, 0.8964, 0.4556, 0.6323],
#          [0.3489, 0.4017, 0.0223, 0.1689, 0.2939]]])

# define the conv
conv1 = torch.nn.Conv1d(3, 3, kernel_size=5, stride=1, padding=2)
torch.nn.init.zeros_(conv1.weight)
torch.nn.init.zeros_(conv1.bias)
# set identity kernel
conv1.weight.data[:, :, 2] = torch.eye(3, 3)

# print mean before
print(x.mean())
# tensor(0.4091)

# print mean after
print(conv1(x).mean())
# tensor(0.4091, grad_fn=<MeanBackward0>)


Instruction: Unable to load model weights while predicting (using pytorch)

Input:
I have trained a Mask RCNN network using PyTorch and am trying to use the obtained weights to predict the location of apples in an image..

I am using the dataset from this paper, and here is the github link to code being used

I am simply following the instructions as provided in the ReadMe file..

Here is the command i wrote in prompt (removed personal info)

  python predict_rcnn.py --data_path "my_directory\datasets\apples-minneapple\detection" --output_file "my_directory\samples\apples\predictions" --weight_file "my_directory\samples\apples\weights\model_19.pth" --mrcnn

model_19.pth is the file with all the weights generated after the 19th epoch

Error is as follows:

Loading model
Traceback (most recent call last):
  File "predict_rcnn.py", line 122, in <module>
    main(args)
  File "predict_rcnn.py", line 77, in main
    model.load_state_dict(checkpoint['model'], strict=False)
KeyError: 'model'

I will paste predict_rcnn.py for convenience:

import os
import torch
import torch.utils.data
import torchvision
import numpy as np

from data.apple_dataset import AppleDataset
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

import utility.utils as utils
import utility.transforms as T

######################################################
# Predict with either a Faster-RCNN or Mask-RCNN predictor
# using the MinneApple dataset
######################################################
def get_transform(train):
    transforms = []
    transforms.append(T.ToTensor())
    if train:
        transforms.append(T.RandomHorizontalFlip(0.5))
    return T.Compose(transforms)

def get_maskrcnn_model_instance(num_classes):
    # load an instance segmentation model pre-trained pre-trained on COCO
    model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=False)

    # get number of input features for the classifier
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    # replace the pre-trained head with a new one
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)

    # now get the number of input features for the mask classifier
    in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
    hidden_layer = 256
    # and replace the mask predictor with a new one
    model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, hidden_layer, num_classes)
    return model

def get_frcnn_model_instance(num_classes):
    # load an instance segmentation model pre-trained pre-trained on COCO
    model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=False)

    # get number of input features for the classifier
    in_features = model.roi_heads.box_predictor.cls_score.in_features
    # replace the pre-trained head with a new one
    model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
    return model

def main(args):
    num_classes = 2
    device = args.device

    # Load the model from
    print("Loading model")
    # Create the correct model type
    if args.mrcnn:
        model = get_maskrcnn_model_instance(num_classes)
    else:
        model = get_frcnn_model_instance(num_classes)

    # Load model parameters and keep on CPU

    checkpoint = torch.load(args.weight_file, map_location=device)
    #checkpoint = torch.load(args.weight_file, map_location=lambda storage, loc: storage)
    model.load_state_dict(checkpoint['model'], strict=False)
    model.eval()

    print("Creating data loaders")
    dataset_test = AppleDataset(args.data_path, get_transform(train=False))
    data_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=1,
                                                   shuffle=False, num_workers=1,
                                                   collate_fn=utils.collate_fn)

    # Create output directory
    base_path = os.path.dirname(args.output_file)
    if not os.path.exists(base_path):
        os.makedirs(base_path)

    # Predict on bboxes on each image
    f = open(args.output_file, 'a')
    for image, targets in data_loader_test:
        image = list(img.to(device) for img in image)
        outputs = model(image)
        for ii, output in enumerate(outputs):
            img_id = targets[ii]['image_id']
            img_name = data_loader_test.dataset.get_img_name(img_id)
            print("Predicting on image: {}".format(img_name))
            boxes = output['boxes'].detach().numpy()
            scores = output['scores'].detach().numpy()

            im_names = np.repeat(img_name, len(boxes), axis=0)
            stacked = np.hstack((im_names.reshape(len(scores), 1), boxes.astype(int), scores.reshape(len(scores), 1)))

            # File to write predictions to
            np.savetxt(f, stacked, fmt='%s', delimiter=',', newline='\n')

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='PyTorch Detection')
    parser.add_argument('--data_path', required=True, help='path to the data to predict on')
    parser.add_argument('--output_file', required=True, help='path where to write the prediction outputs')
    parser.add_argument('--weight_file', required=True, help='path to the weight file')
    parser.add_argument('--device', default='cuda', help='device to use. Either cpu or cuda')
    model = parser.add_mutually_exclusive_group(required=True)
    model.add_argument('--frcnn', action='store_true', help='use a Faster-RCNN model')
    model.add_argument('--mrcnn', action='store_true', help='use a Mask-RCNN model')

    args = parser.parse_args()
    main(args)

Output:
There is no 'model' parameter in the saved checkpoint. If you look in train_rcnn.py:106:

torch.save(model.state_dict(), os.path.join(args.output_dir, 'model_{}.pth'.format(epoch)))

you see that they save just the model parameters. It should've been something like:

torch.save({
    "model": model.state_dict(),
    "optimizer": optimizer.state_dict(),
    "lr_scheduler": lr_scheduler.state_dict()
}, os.path.join(args.output_dir, 'model_{}.pth'.format(epoch)))

so then after loading you get a dictionary with 'model', and the other parameters they appear to be wanting to keep.

This seems to be a bug in their code.


Instruction: Mapping a list of label with one hot encoding

Input:
How can I do the same thing when 'label' is a list?
For example: label = [2,4,6,1,7...,9]

label = 3
NumClass = 10
NumRows  = 100

mask    =  torch.zeros(100, 64)
ones     =  torch.ones(1, 64)
ElementsPerClass = NumRows//NumClass
mask [ ElementsPerClass*label : ElementsPerClass*(label+1) ] = ones

Output:
You are looking for scatter:

NumRows = len(label)
mask = torch.zeros((NumRoes, NumClass)).scatter_(dim=1, index=torch.tensor(label, dtype=torch.long)[:, None], src=torch.ones(NumRows, 1))


Instruction: How to convert a trained model to a function?

Input:
I have finished training a model, what I want to do next is hand over my model to my colleages who know nothing about deep learning, I just want to give them a function that they can run without installing tensorflow or Python on their machines, may be just python (ideally I would love it to run on Matlab). Is this doable? How can I abstract away all or codify everything from them?

I read about deployment of models but it's all about servers and stuff, this is not what I want.
PS: assume a TF model for now.

Output:
Basically, there's onnx that aims at assisting in deploying trained models.
I do not have much experience with it, but I know it's not always a straight forward procedure.


Instruction: Certain members of a torch module aren't moved to GPU even if model.to(device) is called

Input:
An mwe is as follows:

import torch
import torch.nn as nn

class model(nn.Module):
    def __init__(self):
        super(model,self).__init__()
        self.mat = torch.randn(2,2)

    def forward(self,x):
        print('self.mat.device is',self.mat.device)
        x = torch.mv(self.mat,x)
        return x

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
m = model()
m.to(device)

x = torch.tensor([2.,1.])
x = x.to(device)

m(x)

The output is

self.mat.device is cpu

and right after that comes

Traceback (most recent call last):
  File "Z:\cudatest.py", line 21, in <module>
    print(m(x))
  File "E:\Python37\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "Z:\cudatest.py", line 11, in forward
    x = torch.mv(self.mat,x)
RuntimeError: Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mv

The code works fine if I set device = torch.device('cpu'). It seems that the problem is model.mat is not moved to GPU even after m.to(device) is called. Why doesn't whis work? How can I fix this? Please note the following:

Even though this particular example can be fixed by using self.mat = nn.Linear(2,2) and x = self.mat(x) instead, in my original program, I need a temporary tensor to store some data in forward() that is also used in some arithmetics. How can I construct such a tensor and send it to GPU when calling m.to(device)
It is not known in advance whether the computer has GPU or not. Therefore, writing self.mat = self.mat.cuda() is not a good solution for my case.

Output:
pytorch apply Module's methods such as .cpu(), .cuda() and .to() only to sub-modules, parameters and buffers, but NOT to regular class members. pytorch has no way of knowing that self.mat, in your case, is an actual tensor that should be moved around.

Once you decide if your mat should be a parameter or a buffer, simply register it accordingly, e.g.

class model(nn.Module):
    def __init__(self):
        super(model,self).__init__()
        self.register_buffer(name='mat', tensor=torch.randn(2,2))


Instruction: Pytorch DataParallel doesn't work when the model contain tensor operation

Input:
If my model contains only nn.Module layers such as nn.Linear, nn.DataParallel works fine.

x = torch.randn(100,10)

class normal_model(torch.nn.Module):
    def __init__(self):
        super(normal_model, self).__init__()
        self.layer = torch.nn.Linear(10,1)

    def forward(self, x):
        return self.layer(x)

model = normal_model()
model = nn.DataParallel(model.to('cuda:0'))
model(x)

However, when my model contains a tensor operation such as the following

class custom_model(torch.nn.Module):
    def __init__(self):
        super(custom_model, self).__init__()
        self.layer = torch.nn.Linear(10,5)
        self.weight = torch.ones(5,1, device='cuda:0')
    def forward(self, x):
        return self.layer(x) @ self.weight

model = custom_model()
model = torch.nn.DataParallel(model.to('cuda:0'))
model(x)

It gives me the following error

  RuntimeError: Caught RuntimeError in replica 1 on device 1. Original
  Traceback (most recent call last):   File
  "/opt/conda/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py",
  line 60, in _worker
      output = module(*input, **kwargs)   File "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py",
  line 541, in call
      result = self.forward(*input, **kwargs)   File "", line 7, in forward
      return self.layer(x) @ self.weight RuntimeError: arguments are located on different GPUs at
  /pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:277

How to avoid this error when we have some tensor operations in our model?

Output:
I have no experience with DataParallel, but I think it might be because your tensor is not part of the model parameters. You can do this by writing:

torch.nn.Parameter(torch.ones(5,1))

Note that you don't have to move it to the gpu when initializing, because now when you call model.to('cuda:0') this is done automatically.

I can imagine that DataParallel uses the model parameters to move them to the appropriate gpu.

See this answer for more on the difference between a torch tensor and torch.nn.Parameter.

If you don't want the tensor values to be updated by backpropagation during training, you can add requires_grad=False.

Another way that might work is to override the to method, and initialize the tensor in the forward pass:

class custom_model(torch.nn.Module):
    def __init__(self):
        super(custom_model, self).__init__()
        self.layer = torch.nn.Linear(10,5)
    def forward(self, x):
        return self.layer(x) @ torch.ones(5,1, device=self.device)
    def to(self, device: str):
        new_self = super(custom_model, self).to(device)
        new_self.device = device
        return new_self

or something like this:

class custom_model(torch.nn.Module):
    def __init__(self, device:str):
        super(custom_model, self).__init__()
        self.layer = torch.nn.Linear(10,5)
        self.weight = torch.ones(5,1, device=device)
    def forward(self, x):
        return self.layer(x) @ self.weight
    def to(self, device: str):
        new_self = super(custom_model, self).to(device)
        new_self.device = device
        new_self.weight = torch.ones(5,1, device=device)
        return new_self


Instruction: how to get alignment or attention information for translations produced by a torch hub model?

Input:
The torch hub provides pretrained models, such as: https://pytorch.org/hub/pytorch_fairseq_translation/
These models can be used in python, or interactively with a CLI.
With the CLI it is possible to get alignments, with the --print-alignment flag. The following code works in a terminal, after installing fairseq (and pytorch)
curl https://dl.fbaipublicfiles.com/fairseq/models/wmt14.v2.en-fr.fconv-py.tar.bz2 | tar xvjf -
MODEL_DIR=wmt14.en-fr.fconv-py
fairseq-interactive \
    --path $MODEL_DIR/model.pt $MODEL_DIR \
    --beam 5 --source-lang en --target-lang fr \
    --tokenizer moses \
    --bpe subword_nmt --bpe-codes $MODEL_DIR/bpecodes \
    --print-alignment

In python it is possible to specify the keyword args verbose and print_alignment:
import torch

en2fr = torch.hub.load('pytorch/fairseq', 'transformer.wmt14.en-fr', tokenizer='moses', bpe='subword_nmt')

fr = en2fr.translate('Hello world!', beam=5, verbose=True, print_alignment=True)

However, this will only output the alignment as a logging message. And for fairseq 0.9 it seems to be broken and results in an error message (issue).
Is there a way to access alignment information (or possibly even the full attention matrix) from python code?

Output:
I've browsed the fairseq codebase and found a hacky way to output alignment information.
Since this requires editing the fairseq sourcecode itself, I don't think it's an acceptable solution. But maybe it helps someone (I'm still very interested in an answer on how to do this properly).

Edit the sample() function and rewrite the return statement.
Here is the whole function (to help you find it better, in the code), but only the last line should be changed:

def sample(self, sentences: List[str], beam: int = 1, verbose: bool = False, **kwargs) -> List[str]:
    if isinstance(sentences, str):
        return self.sample([sentences], beam=beam, verbose=verbose, **kwargs)[0]
    tokenized_sentences = [self.encode(sentence) for sentence in sentences]
    batched_hypos = self.generate(tokenized_sentences, beam, verbose, **kwargs)
    return list(zip([self.decode(hypos[0]['tokens']) for hypos in batched_hypos], [hypos[0]['alignment'] for hypos in batched_hypos]))


Instruction: How to make a 3D tensor into one_hot encoding

Input: for example ,labels is a groundtruth, labels size is [4,224,224] where 4 means batchsize ,224 means h and w,  labels dtype is torch.int64, in my training code ,label's pixel from 0 to 255 .I use my network to Semantic segmentation

Output:
From your question, it seems like you would like 256 integer labels, from 0 to 255. This can be done by the following:

# let a be the (4, 224, 224) dim tensor
labels = np.stack([(a == i).int() for i in range(256)])
print(labels.shape) #(256, 4, 224, 224) dimensional one-hot encoding

Hope this helps!


Instruction: Using WeightedRandomSampler in PyTorch

Input:
I need to implement a multi-label image classification model in PyTorch. However my data is not balanced, so I used the WeightedRandomSampler in PyTorch to create a custom dataloader. But when I iterate through the custom dataloader, I get the error : IndexError: list index out of range

Implemented the following code using this link :https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/3?u=surajsubramanian

def make_weights_for_balanced_classes(images, nclasses):
    count = [0] * nclasses
    for item in images:
        count[item[1]] += 1
    weight_per_class = [0.] * nclasses
    N = float(sum(count))
    for i in range(nclasses):
        weight_per_class[i] = N/float(count[i])
    weight = [0] * len(images)
    for idx, val in enumerate(images):
        weight[idx] = weight_per_class[val[1]]
    return weight

weights = make_weights_for_balanced_classes(train_dataset.imgs, len(full_dataset.classes))
weights = torch.DoubleTensor(weights)
sampler = WeightedRandomSampler(weights, len(weights))

train_loader = DataLoader(train_dataset, batch_size=4,sampler = sampler, pin_memory=True)

Based on the answer in https://stackoverflow.com/a/60813495/10077354, the following is my updated code. But then too when I create a dataloader :loader = DataLoader(full_dataset, batch_size=4, sampler=sampler), len(loader) returns 1.

class_counts = [1691, 743, 2278, 1271]
num_samples = np.sum(class_counts)
labels = [tag for _,tag in full_dataset.imgs]

class_weights = [num_samples/class_counts[i] for i in range(len(class_counts)]
weights = [class_weights[labels[i]] for i in range(num_samples)]
sampler = WeightedRandomSampler(torch.DoubleTensor(weights), num_samples)

Thanks a lot in advance !

I included an utility function based on the accepted answer below :

def sampler_(dataset):
    dataset_counts = imageCount(dataset)
    num_samples = sum(dataset_counts)
    labels = [tag for _,tag in dataset]

    class_weights = [num_samples/dataset_counts[i] for i in range(n_classes)]
    weights = [class_weights[labels[i]] for i in range(num_samples)]
    sampler = WeightedRandomSampler(torch.DoubleTensor(weights), int(num_samples))
    return sampler

The imageCount function finds number of images of each class in the dataset. Each row in the dataset contains the image and the class, so we take the second element in the tuple into consideration.

def imageCount(dataset):
    image_count = [0]*(n_classes)
    for img in dataset:
        image_count[img[1]] += 1
    return image_count

Output:
That code looks a bit complex... You can try the following:

#Let there be 9 samples and 1 sample in class 0 and 1 respectively
class_counts = [9.0, 1.0]
num_samples = sum(class_counts)
labels = [0, 0,..., 0, 1] #corresponding labels of samples

class_weights = [num_samples/class_counts[i] for i in range(len(class_counts))]
weights = [class_weights[labels[i]] for i in range(int(num_samples))]
sampler = WeightedRandomSampler(torch.DoubleTensor(weights), int(num_samples))


Instruction: How to load all 5 batches of CIFAR10 in a single data structure as on MNIST in PyTorch?

Input:
With Mnist I have a single file with the labels and a single file for the train, so I simply do:

self.data = datasets.MNIST(root='./data', train=True, download=True)

Basically I create a set of labels (from 0-9) and save the i-th position of the image in the data structure, to create my custom tasks:

   def make_tasks (self):
            self.task_to_examples = {} #task 0-9
            self.all_tasks = set (self.data.train_labels.numpy ())
            for i, digit in enumerate (self.data.train_labels.numpy ()):
                if str(digit) not in self.task_to_examples:
                    self.task_to_examples[str(digit)] = []
                self.task_to_examples[str(digits)].append(i)

I don't understand how to do the same thing using CIFAR10 because it is divided into 5 batches, I would like all the data in a single structure.

Output:
If your desired structure is {"class_id": [indices of the samples]}, then for CIFAR10 you can do something like this:

import numpy as np
import torchvision

# set root accordingly
cifar = torchvision.datasets.CIFAR10(root=".", train=True, download=True)
task_to_examples = {
    str(task_id): np.where(cifar.targets == task_id)[0].tolist()
    for task_id in np.unique(cifar.targets)
}


Instruction: Scope of with torch.no_grad() in pytorch

Input:
with torch.no_grad():
    input = Variable(input).cuda()
    target = Variable(target).cuda(non_blocking=True)
y=model(input)
# many things here

Is the no_grad continue to having effect out of the "with" scope?

Output:
The no_grad has no effect outside the "with" scope.

According to this answer from a moderator on the pytorch blog:

with torch.no_grad():
    # No gradients in this block
    x = self.cnn(x)

# Gradients as usual outside of it
x = self.lstm(x)

It is the purpose of the with statement in python. The variable used  by the with (here torch.no_grad()) has only effect in the with context and not after. See the python doc for complete details.


Instruction: Find indices of elements equal to zero in a PyTorch Tensor

Input:
My question is nearly identical to this one, with the notable difference of being in PyTorch. I would prefer not to use the Numpy solution as this would involve moving data back to the CPU. I see that, as with Numpy, PyTorch has a nonzero function, however its where function (the solution in the Numpy thread I linked) has behavior different from Numpy's.
The behavior I want is an is_zero() function as follows:

>>> arr.nonzero()
tensor([[0, 1],
        [1, 0]])
>>> arr.is_zero()
tensor([[0, 0],
        [1, 1]])

Output:
You can make a boolean mask and then call nonzero():

(arr == 0).nonzero()

For instance:

arr = torch.randint(high=2, size=(3, 3))

tensor([[1, 1, 0],  # (0, 2)
        [1, 1, 0],  # (1, 2)
        [1, 0, 0]]) # (2, 1) and (2, 2)

(arr == 0).nonzero()

tensor([[0, 2],
        [1, 2],
        [2, 1],
        [2, 2]])


Instruction: Implementing a simple ResNet block with PyTorch

Input:
I'm trying to implement following ResNet block, which ResNet consists of blocks with two convolutional layers and a skip connection. For some reason it doesn't add the output of skip connection, if applied, or input to the output of convolution layers.

The ResNet block has:

Two convolutional layers with:

3x3 kernel
no bias terms
padding with one pixel on both sides
2d batch normalization after each convolutional layer

The skip connection:

simply copies the input if the resolution and the number of channels do not change.
if either the resolution or the number of channels change, the skip connection should have one convolutional layer with:

1x1 convolution without bias
change of the resolution with stride (optional)
different number of input channels and output channels (optional)
the 1x1 convolutional layer is followed by 2d batch normalization.

The ReLU nonlinearity is applied after the first convolutional layer and at the end of the block.

My code:

class Block(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1):
        """
        Args:
          in_channels (int):  Number of input channels.
          out_channels (int): Number of output channels.
          stride (int):       Controls the stride.
        """
        super(Block, self).__init__()

        self.skip = nn.Sequential()

        if stride != 1 or in_channels != out_channels:
          self.skip = nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride, bias=False),
            nn.BatchNorm2d(out_channels))
        else:
          self.skip = None

        self.block = nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1, stride=1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(),
            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=3, padding=1, stride=1, bias=False),
            nn.BatchNorm2d(out_channels))

    def forward(self, x):
        out = self.block(x)

        if self.skip is not None:
          out = self.skip(x)
        else:
          out = x

        out += x

        out = F.relu(out)
        return out

Output:
The problem is in the reuse of the out variable. Normally, you'd implement like this:

def forward(self, x):
    identity = x
    out = self.block(x)

    if self.skip is not None:
        identity = self.skip(x)

    out += identity
    out = F.relu(out)

    return out

If you like "one-liners":

def forward(self, x):
    out = self.block(x)
    out += (x if self.skip is None else self.skip(x))
    out = F.relu(out)
    return out

If you really like one-liners (please, that is too much, do not choose this option :))

def forward(self, x):
    return F.relu(self.block(x) + (x if self.skip is None else self.skip(x)))


Instruction: Efficiently saving and loading data using h5py (or other methods)

Input:
I am testing ways of efficient saving and retrieving data using h5py. But am having trouble with running time while not using up all my memory.

In my first method I simply create a static h5py file

with h5py.File(fileName, 'w') as f:
        f.create_dataset('data_X', data = X, dtype = 'float32')
        f.create_dataset('data_y', data = y, dtype = 'float32')

In the second method, I set parameter maxshape in order to append more training data in the future. (see How to append data to one specific dataset in a hdf5 file with h5py)

with h5py.File(fileName2, 'w') as f:
            f.create_dataset('data_X', data = X, dtype = 'float32',maxshape=(None,4919))
            f.create_dataset('data_y', data = y, dtype = 'float32',maxshape=(None,6))

I am using PyTorch and am set up my data loader as such:

class H5Dataset_all(torch.utils.data.Dataset):
    def __init__(self, h5_path):
        # super(dataset_h5, self).__init__()
        self.h5_path = h5_path
        self._h5_gen = None

    def __getitem__(self, index):
        if self._h5_gen is None:
            self._h5_gen = self._get_generator()
            next(self._h5_gen)
        return self._h5_gen.send(index)

    def _get_generator(self):
        with h5py.File( self.h5_path, 'r') as record:
            index = yield
            while True:
                X = record['data_X'][index]
                y = record['data_y'][index]
                index = yield X, y

    def __len__(self):
        with h5py.File(self.h5_path,'r') as record:
            length = record['data_X'].shape[0]
            return length

loader = Data.DataLoader(
        dataset=H5Dataset_all(filename),
        batch_size=BATCH_SIZE,
        shuffle=True, num_workers=0)

Having saved the same data for each of these methods I would expect them to be similar in running time, however that is not the case. The data I used has size X.shape=(200722,4919) and y.shape=(200772,6). The files are about 3.6 GB each.
I test the running time using:

import time
t0 = time.time()
for i, (X_batch, y_batch) in enumerate(loader):
    # assign a dummy value
    a = 0
t1 = time.time()-t0
print(f'time: {t1}')

For the first method the running time is 83 s and for the second it is 1216 s, which In my mind doesn't make sense. Can anyone help me figure out why?

Additionally I also tried saving/loading it as a torch file using torch.save and torch.load and passing the data to Data.TensorDataset before setting up the loader. This implementation runs significantly faster (about 3.7 s), but has the disadvantage of having to load the files before training, which could quickly be capped by my memory.

Is there a better way in which I can train somewhat fast while not using having to load all of the data before training?

Output:
This looks like an I/O performance issue. To test, I created a very simple example to compare your 2 methods. (My code is at the end of the post.) I found the exact opposite behavior (assuming my code mimics your process). Writing the dataset is slower when I don't use the maxshape=() parameter: 62 sec to create w/out maxshape versus 16 sec to create with maxshape. To verify the operations aren't order dependent, I also ran creating _2 first, then created _1, and got very similar results.
Here is the timing data:

create data_X time: 62.60318350791931
create data_y time: 0.010000228881835
** file 1 Done **

create data_X time: 16.416041135787964
create data_y time: 0.0199999809265136
** file 2 Done **

Code to create the 2 files below:

import h5py
import numpy as np
import time

n_rows = 200722
X_cols = 4919
y_cols = 6

X = np.random.rand(n_rows,X_cols).astype('float32')
y = np.random.rand(n_rows,y_cols).astype('float32')

t0 = time.time()
with h5py.File('SO_60818355_1.h5', 'w') as h5f:
     h5f.create_dataset('data_X', data = X)
     t1 = time.time()
     print(f'create data_X time: {t1-t0}')

     h5f.create_dataset('data_y', data = y)
     t2 = time.time()
     print(f'create data_y time: {t2-t1}')
print ('** file 1 Done ** \n ')

t0 = time.time()
with h5py.File('SO_60818355_2.h5', 'w') as h5f:
     h5f.create_dataset('data_X', data = X, maxshape=(None,X_cols))
     t1 = time.time()
     print(f'create data_X time: {t1-t0}')

     h5f.create_dataset('data_y', data = y, maxshape=(None,y_cols))
     t2 = time.time()
     print(f'create data_y time: {t2-t1}')
print ('** file 2 Done ** \n ')


Instruction: list of tensors into tensors

Input:
I have a list like

xs = [tensor1, tensor2, tensor3....]

I want to change it into a tensor so that I can feed xs and ys into the Pytorch DataLoader.
I tried this,

xs = torch.Tensor(xs)

but it doesn't work as the individual elements are not items but tensors.

Output:
You may want torch.stack

xs = torch.stack(xs)


Instruction: Splitting ImageFolder into train and validation datasets

Input:
I have loaded my dataset as follows :

full_dataset = ImageFolder(root = os.path.join(root, 'train'), transform=train_transforms)

Now to split my dataset into training and validation sets, I used the following code :

train_size = int(0.8 * len(full_dataset))
validation_size = len(full_dataset) - train_size
train_dataset, validation_dataset = random_split(full_dataset, [train_size, validation_size])

Both train_dataset and validation_dataset are of type : torch.utils.data.dataset.Subset. Is there any way to convert these datasets into torchvision.datasets.folder.ImageFolder.
I need to do this as I am not able to iterate through datasets of type torch.utils.data.dataset.Subset

Output:
You should be able to iterate through a Subset just fine, since it has the __getitem__ method implemented as you can see from the source code :

class Subset(Dataset):
    r"""
    Subset of a dataset at specified indices.

    Arguments:
        dataset (Dataset): The whole Dataset
        indices (sequence): Indices in the whole set selected for subset
    """
    def __init__(self, dataset, indices):
        self.dataset = dataset
        self.indices = indices

    def __getitem__(self, idx):
        return self.dataset[self.indices[idx]]

    def __len__(self):
        return len(self.indices)

So the following should work:

for image, label in train_dataset:
    print(image, label)

Or you can create a Dataloader from a Subset:

train_dataloader = DataLoader(train_dataset, batch_size, shuffle)

for images, labels in train_dataloader:
    print(images, labels)

Same for validation_dataset.


Instruction: CUDA kernel failed : no kernel image is available for execution on the device, Error when running PyTorch model inside Google Compute VM

Input:
I have a docker image of a PyTorch model that returns this error when run inside a google compute engine VM running on debian/Tesla P4 GPU/google deep learning image:

CUDA kernel failed : no kernel image is available for execution on the device

This occurs on the line where my model is called. The PyTorch model includes custom c++ extensions, I'm using this model https://github.com/daveredrum/Pointnet2.ScanNet

My image installs these at runtime

The image runs fine on my local system. Both VM and my system have these versions:

Cuda compilation tools    10.1, V10.1.243

torch                     1.4.0

torchvision               0.5.0

The main difference is the GPU as far as I'm aware

Local:

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 435.21       Driver Version: 435.21       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 960M    Off  | 00000000:01:00.0 Off |                  N/A |
| N/A   36C    P8    N/A /  N/A |    361MiB /  2004MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

VM:

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   42C    P0    23W /  75W |      0MiB /  7611MiB |      3%      Default |

If I ssh into the VM torch.cuda.is_available() returns true

Therefore I suspect it must be something to do with the compilation of the extensions

This is the relevant part of my docker file:

ENV CUDA_HOME "/usr/local/cuda-10.1"
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda-10.1/bin:${PATH}
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419"
ENV FORCE_CUDA=1

# CUDA 10.1-specific steps
RUN conda install -c open3d-admin open3d
RUN conda install -y -c pytorch \
    cudatoolkit=10.1 \
    "pytorch=1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0" \
    "torchvision=0.5.0=py36_cu101" \
 && conda clean -ya
RUN pip install -r requirements.txt
RUN pip install flask
RUN pip install plyfile
RUN pip install scipy

# Install OpenCV3 Python bindings
RUN sudo apt-get update && sudo apt-get install -y --no-install-recommends \
    libgtk2.0-0 \
    libcanberra-gtk-module \
    libgl1-mesa-glx \
 && sudo rm -rf /var/lib/apt/lists/*

RUN dir
RUN cd pointnet2 && python setup.py install
RUN cd ..

I have already re-running this line from ssh in the VM:

TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0" python setup.py install

Which I think targets the installation to the Tesla P4 compute capability?

Is there some other setting or troubleshooting step I can try?

I didn't know anything about docker/VMs/pytorch extensions until a couple of days ago, so somewhat shooting in the dark. Also this is my first stackoverflow post, apologies if I'm not following some etiquette, feel free to point out.

Output:
I resolved this in the end by manually deleting all the folders except for "src" in the folder containing setup.py

Then rebuilt the docker image

Then when building the image I ran TORCH_CUDA_ARCH_LIST="6.1" python setup.py install, to install the cuda extensions targeting the correct compute capability for the GPU on the VM

and it worked!

I guess just running setup.py without deleting the folders previously installed doesn't fully overwrite the extension


Instruction: Why can't I append a PyTorch tensor with torch.cat?

Input:
I have:

import torch

input_sliced = torch.rand(180, 161)
output_sliced = torch.rand(180,)

batched_inputs = torch.Tensor()
batched_outputs = torch.Tensor()

print('input_sliced.size', input_sliced.size())
print('output_sliced.size', output_sliced.size())

batched_inputs = torch.cat((batched_inputs, input_sliced))
batched_outputs = torch.cat((batched_outputs, output_sliced))

print('batched_inputs.size', batched_inputs.size())
print('batched_outputs.size', batched_outputs.size())

This outputs:

input_sliced.size torch.Size([180, 161])
output_sliced.size torch.Size([180])

batched_inputs.size torch.Size([180, 161])
batched_outputs.size torch.Size([180])

I need the batched ones to append, but the torch.cat isn't working. What am I doing wrong?

Output:
Assuming you're doing it in a loop, I'd say it is better to do like this:

import torch

batch_input, batch_output = [], []
for i in range(10):  # assuming batch_size=10
    batch_input.append(torch.rand(180, 161))
    batch_output.append(torch.rand(180,))

batch_input = torch.stack(batch_input)
batch_output = torch.stack(batch_output)

print(batch_input.shape)   # output: torch.Size([10, 180, 161])
print(batch_output.shape)  # output: torch.Size([10, 180])

If you know the resulting batch_* shape a priori, you can preallocate the final Tensor and simply assign each sample into their corresponding positions in the batch. It would be more memory efficient.


Instruction: How to test one single image in pytorch

Input:
I created my model in pytorch and is working really good, but when i want to test just one image batch_size=1 always return the second class (in this case a dog).

I tried to test with batch > 1 and in all cases this works!

The architecture:

model = models.densenet121(pretrained=True)

for param in model.parameters():
param.requires_grad = False
from collections import OrderedDict
classifier = nn.Sequential(OrderedDict([
                          ('fc1', nn.Linear(1024, 500)),
                          ('relu', nn.ReLU()),
                          ('fc2', nn.Linear(500, 2)),
                          ('output', nn.LogSoftmax(dim=1))
                          ]))

model.classifier = classifier

so my tensors are [batch, 3, 224, 224]

i have tried with:

resize
reshape
unsqueeze(0)

the response when is one image is always [[0.4741, 0.5259]]

My Test Code

from PIL import *
msize = 256
loader = transforms.Compose([transforms.Scale(imsize), transforms.ToTensor()])

def image_loader(image_name):
    """load image, returns cuda tensor"""
    image = Image.open(image_name)
    image = loader(image).float()
    image = image.unsqueeze(0)
    return image.cuda()
image = image_loader('Cat_Dog_data/test/cat/cat.16.jpg')
with torch.no_grad():
    logits = model.forward(image)
ps = torch.exp(logits)
_, predTest = torch.max(ps,1)
print(ps) ## same value in all cases
imagen_mostrar = images[ii].to('cpu')
helper.imshow(imagen_mostrar,title=clas_perro_gato(predTest), normalize=True)

Second Test Code

andrea_data = datasets.ImageFolder(data_dir + '/andrea', transform=test_transforms)
andrealoader = torch.utils.data.DataLoader(andrea_data, batch_size=1, shuffle=True)
dataiter = iter(andrealoader)
images, labels = dataiter.next()
images, labels = images.to(device), labels.to(device)
ps = torch.exp(model.forward(images))
_, predTest = torch.max(ps,1)
print(ps.float())

if i changed my batch_size to 1 always returned a tensor who say that is a dog [0.43,0.57] for example.

Thanks!

Output:
I realized that my model wasn't in eval mode.

So i just added model.eval() and now that's all, works for any size batch


Instruction: How to define ration of summary with hugging face transformers pipeline?

Input:
I am using the following code to summarize an article from using huggingface-transformer's pipeline. Using this code:

from transformers import pipeline
summarizer = pipeline(task="summarization" )
summary = summarizer(text)
print(summary[0]['summary_text'])

How can I define a ratio between the summary and the original article? For example, 20% of the original article?

EDIT 1: I implemented the solution you suggested, but got the following error. This is the code I used:

summarizer(text, min_length = int(0.1 * len(text)), max_length = int(0.2 * len(text)))
print(summary[0]['summary_text'])

The error I got:

RuntimeError                              Traceback (most recent call last)
<ipython-input-9-bc11c5d8eb66> in <module>()
----> 1 summarizer(text, min_length = int(0.1 * len(text)), max_length = int(0.2 * len(text)))
      2 print(summary[0]['summary_text'])

13 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1482         # remove once script supports set_grad_enabled
   1483         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1484     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1485
   1486

RuntimeError: index out of range: Tried to access index 1026 out of table with 1025 rows. at /pytorch/aten/src/TH/generic/THTensorEvenMoreMath.cpp:418

Output:
(Note that this answer is based on the documentation for version 2.6 of transformers)

It seems that as of yet the documentation on the pipeline feature is still very shallow, which is why we have to dig a bit deeper. When calling a Python object, it internally references its own __call__ property, which we can find here for the summarization pipeline.

Note that it allows us (similar to the underlying BartForConditionalGeneration model) to specifiy the min_length and max_length, which is why we can simply call with something like

summarizer(text, min_length = 0.1 * len(text), max_length = 0.2 * len(text)

This would give you a summary of about 10-20% length of the original data, but of course you can change that to your liking. Note that the default value for BartForConditionalGeneration for max_length is 20 (as of now, min_length is undocumented, but defaults to 0), whereas the summarization pipeline has values min_length=21 and max_length=142.


Instruction: PyTorch - convert ProGAN agent from pth to onnx

Input:
I trained a ProGAN agent using this PyTorch reimplementation, and I saved the agent as a .pth. Now I need to convert the agent into the .onnx format, which I am doing using this scipt:

from torch.autograd import Variable

import torch.onnx
import torchvision
import torch

device = torch.device("cuda")

dummy_input = torch.randn(1, 3, 64, 64)
state_dict = torch.load("GAN_agent.pth", map_location = device)

torch.onnx.export(state_dict, dummy_input, "GAN_agent.onnx")

Once I run it, I get the error AttributeError: 'collections.OrderedDict' object has no attribute 'state_dict' (full prompt below). As far as I understood, the problem is that converting the agent into .onnx requires more information. Am I missing something?

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-2-c64481d4eddd> in <module>
     10 state_dict = torch.load("GAN_agent.pth", map_location = device)
     11
---> 12 torch.onnx.export(state_dict, dummy_input, "GAN_agent.onnx")

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\__init__.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
    146                         operator_export_type, opset_version, _retain_param_name,
    147                         do_constant_folding, example_outputs,
--> 148                         strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
    149
    150

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
     64             _retain_param_name=_retain_param_name, do_constant_folding=do_constant_folding,
     65             example_outputs=example_outputs, strip_doc_string=strip_doc_string,
---> 66             dynamic_axes=dynamic_axes, keep_initializers_as_inputs=keep_initializers_as_inputs)
     67
     68

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, example_outputs, propagate, opset_version, _retain_param_name, do_constant_folding, strip_doc_string, dynamic_axes, keep_initializers_as_inputs, fixed_batch_size)
    414                                                         example_outputs, propagate,
    415                                                         _retain_param_name, do_constant_folding,
--> 416                                                         fixed_batch_size=fixed_batch_size)
    417
    418         # TODO: Don't allocate a in-memory string for the protobuf

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _model_to_graph(model, args, verbose, training, input_names, output_names, operator_export_type, example_outputs, propagate, _retain_param_name, do_constant_folding, _disable_torch_constant_prop, fixed_batch_size)
    277             model.graph, tuple(in_vars), False, propagate)
    278     else:
--> 279         graph, torch_out = _trace_and_get_graph_from_model(model, args, training)
    280         state_dict = _unique_state_dict(model)
    281         params = list(state_dict.values())

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _trace_and_get_graph_from_model(model, args, training)
    226     # A basic sanity check: make sure the state_dict keys are the same
    227     # before and after running the model.  Fail fast!
--> 228     orig_state_dict_keys = _unique_state_dict(model).keys()
    229
    230     # By default, training=False, which is good because running a model in

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\jit\__init__.py in _unique_state_dict(module, keep_vars)
    283     # id(v) doesn't work with it. So we always get the Parameter or Buffer
    284     # as values, and deduplicate the params using Parameters and Buffers
--> 285     state_dict = module.state_dict(keep_vars=True)
    286     filtered_dict = type(state_dict)()
    287     seen_ids = set()

AttributeError: 'collections.OrderedDict' object has no attribute 'state_dict'

Output:
Files you have there are state_dict, which are simply mappings of layer name to tensor weights biases and a-like (see here for more thorough introduction).

What that means is that you need a model so those saved weights and biases can be mapped upon, but first things first:

1. Model preparation

Clone the repository where model definitions are located and open file /pro_gan_pytorch/pro_gan_pytorch/PRO_GAN.py. We need some modifications in order for it to work with onnx. onnx exporter requires input to be passed as torch.tensor only (or list/dict of those), while Generator class needs int and float arguments).

Simple solution it to slightly modify forward function (line 80 in the file, you can verify it on GitHub) to the following:

def forward(self, x, depth, alpha):
    """
    forward pass of the Generator
    :param x: input noise
    :param depth: current depth from where output is required
    :param alpha: value of alpha for fade-in effect
    :return: y => output
    """

    # THOSE TWO LINES WERE ADDED
    # We will pas tensors but unpack them here to `int` and `float`
    depth = depth.item()
    alpha = alpha.item()
    # THOSE TWO LINES WERE ADDED
    assert depth < self.depth, "Requested output depth cannot be produced"

    y = self.initial_block(x)

    if depth > 0:
        for block in self.layers[: depth - 1]:
            y = block(y)

        residual = self.rgb_converters[depth - 1](self.temporaryUpsampler(y))
        straight = self.rgb_converters[depth](self.layers[depth - 1](y))

        out = (alpha * straight) + ((1 - alpha) * residual)

    else:
        out = self.rgb_converters[0](y)

    return out

Only unpacking via item() was added here. Every input which is not of Tensor type should be packed as one in function definition and unpacked ASAP at the top of your function. It will not destroy your created checkpoint so no worries as it's just layer-weight mapping.

2. Model exporting

Place this script in /pro_gan_pytorch (where README.md is located as well):

import torch

from pro_gan_pytorch import PRO_GAN as pg

gen = torch.nn.DataParallel(pg.Generator(depth=9))
gen.load_state_dict(torch.load("GAN_GEN_SHADOW_8.pth"))

module = gen.module.to("cpu")

# Arguments like depth and alpha may need to be changed
dummy_inputs = (torch.randn(1, 512), torch.tensor([5]), torch.tensor([0.1]))
torch.onnx.export(module, dummy_inputs, "GAN_GEN8.onnx", verbose=True)

Please notice a few things:

We have to create model before loading weights as it's a state_dict only.
torch.nn.DataParallel is needed as that's what the model was trained on (not sure about your case, please adjust accordingly). After loading we can get the module itself via module attribute.
everything is casted to CPU, no need for GPU here I think. You could cast everything to GPU if you so insist though.
dummy input to generator cannot be an image (I used files provided by repo authors on their Google Drive), it has to be noise with 512 elements.

Run it and your .onnx file should be there.

Oh, and as you are after different checkpoint you may want to follow similar procedure, though no guarantees everything will work fine (it does look like it though).


Instruction: Getting error when trying to torch stack a list of tensors

Input:
I am trying to get cuda to work but I need to change my training input into a tensor. When I tried to do that, I am getting an error when I tried to stack a list of tensor into one tensor.

Code

for epoch in:
        alst = []
        for x, y in loader:
            x = torch.stack(x)
            #x = torch.Tensor(x)
            #x = torch.stack(x).to(device,dtype=float)

Shape of x: List of tensors

[tensor([[[0.325],
     [ 0.1257],
     [ 0.1149],
     ...,
     [-1.572],
     [-1.265],
     [-3.574]],
]), tensor([1,2,3,4,5]), tensor(6,5,4,3,2])]

Error I got

     22             alst = []
     23             for x, y in loader:
---> 24                 x_list = torch.stack(x)
     25 #                 x = torch.Tensor(x)
     26 #                 x = torch.stack(x).to(device,dtype=float)

RuntimeError: Expected object of scalar type Float but got scalar type Long for sequence element 1 in sequence argument at position #1 'tensors'

Not sure what I am doing wrong. I tried x = torch.stack(x).to(device,dtype=float) as well but it still didn't work.

Output:
First tensor in your output is of float type with values to input your network with, second looks like labels (of type long).

Furthermore the first one is a tensor while the second and third element are vectors (with 6 and 9 elements respectively).

You cannot stack tensors of different shape hence this won't work no matter the types.

Unpack your x via

matrix, vector1, vector2 = x

To remove type warnings cast vector1 and vector2 to float via

vector1 = vector1.float()

Check their shapes via .shape attribute and act accordingly.
Probably though you already have batches of data as you are using train_loader. See: DataLoader documentation for more information and check whether you are using one.


Instruction: AttributeError: 'torch.return_types.max' object has no attribute 'dim' - Maxpooling Channel

Input:
I'm trying to do maxpooling over channel dimension:

class ChannelPool(nn.Module):
    def forward(self, input):
        return torch.max(input, dim=1)

but I get the error

AttributeError: 'torch.return_types.max' object has no attribute 'dim'

Output:
The torch.max function called with dim returns a tuple so:
class ChannelPool(nn.Module):
    def forward(self, input):
        input_max, max_indices = torch.max(input, dim=1)
        return input_max

From the documentation of torch.max:

Returns a namedtuple (values, indices) where values is the maximum value of each row of the input tensor in the given dimension dim. And indices is the index location of each maximum value found (argmax).


Instruction: Confusion in understanding the output of BERTforTokenClassification class from Transformers library

Input:
It is the example given in the documentation of transformers pytorch library

from transformers import BertTokenizer, BertForTokenClassification
import torch

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForTokenClassification.from_pretrained('bert-base-uncased',
                      output_hidden_states=True, output_attentions=True)

input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute",
                         add_special_tokens=True)).unsqueeze(0)  # Batch size 1
labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
outputs = model(input_ids, labels=labels)

loss, scores, hidden_states,attentions = outputs

Here hidden_states is a tuple of length 13 and contains hidden-states of the model at the output of each layer plus the initial embedding outputs. I would like to know, whether hidden_states[0] or hidden_states[12] represent the final hidden state vectors?

Output:
If you check the source code, specifically BertEncoder, you can see that the returned states are initialized as an empty tuple and then simply appended per iteration of each layer.
The final layer is appended as the last element after this loop, see here, so we can safely assume that hidden_states[12] is the final vectors.


Instruction: Hyper-parameter tuning and Over-fitting with Feed-Forward Neural Network - Mini-Batch Epoch and Cross Validation

Input:
I am looking at implementing a hyper-parameter tuning method for a feed-forward neural network (FNN) implemented using PyTorch. My original FNN , the model is named net, has been implemented using a mini-batch learning approach with epochs:

#Parameters
batch_size = 50 #larger batch size leads to over fitting
num_epochs = 1000
learning_rate = 0.01 #was .01-AKA step size - The amount that the weights are updated during training
batch_no = len(x_train) // batch_size

criterion = nn.CrossEntropyLoss()  #performance of a classification model whose output is a probability value between 0 and 1
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

for epoch in range(num_epochs):
    if epoch % 20 == 0:
        print('Epoch {}'.format(epoch+1))
    x_train, y_train = shuffle(x_train, y_train)
    # Mini batch learning - mini batch since batch size < n(batch gradient descent), but > 1 (stochastic gradient descent)
    for i in range(batch_no):
        start = i * batch_size
        end = start + batch_size
        x_var = Variable(torch.FloatTensor(x_train[start:end]))
        y_var = Variable(torch.LongTensor(y_train[start:end]))
        # Forward + Backward + Optimize
        optimizer.zero_grad()
        ypred_var = net(x_var)
        loss =criterion(ypred_var, y_var)
        loss.backward()
        optimizer.step()

I lastly test my model on a separate test set.

I came across an approach using randomised search to tune the hyper-parameters as well as implementing K-fold cross-validation (RandomizedSearchCV).

My question is two-fold(no pun intended!) and firstly is theoretical: Is k-fold validation is necessary or could add any benefit to mini-batch feed-forward neural network? From what I can see, the mini-batch approach should do roughly the same job, stopping over-fitting.

I also found a good answer here but I'm not sure this addresses a mini-batch approach approach specifically.

Secondly, if k-fold is not necessary, is there another hyper-parameter tuning function for PyTorch to avoid manually creating one?

Output:
k-fold cross validation is generally useful when you have a very small dataset. Thus, if you are training on a dataset like CIFAR10 (which is large, 60000 images), then you don't require k-fold cross validation.
The idea of k-fold cross validation is to see how model performance (generalization) varies as different subsets of data is used for training and testing. This becomes important when you have very less data. However, for large datasets, the metric results on the test dataset is enough to test the generalization of the model.
Thus, whether you require k-fold cross validation depends on the size of your dataset. It does not depend on what model you use.
If you look at this chapter of the Deep Learning book (this was first referenced in this link):

  Small batches can oﬀer a regularizing eﬀect (Wilson and Martinez, 2003), perhaps due to the noise they add to the learning process. Generalization error is often best for a batch size of 1. Training with such a small batch size might require a small learning rate to maintain stability because of the high variance in the estimate of the gradient. The total runtime can be very high as a result of the need to make more steps, both because of the reduced learning rate and because it takes more steps to observe the entire training set.

So, yes, mini-batch training will have a regularizing effect (reduce overfitting) to some extent.
There is no inbuilt hyperparameter tuning (at least at the time of writing this answer), but many developers have developed tools for this purpose (for example). You can find more such tools by searching for them. This question has answers which list a lot of such tools.


Instruction: Return all dimensions if there are 3 channels, else return 0

Input:
I want to plot some images that are sometimes grayscale, sometimes color.

Since I'm using pytorch, the transformed images are either (50, 100, 1) or (50, 100, 3) depending on if they're color or not. Since matplotlib cannot take the former for a picture, I need to return these shapes:

(100, 100, 3) # if it's a color picture
(100, 100) # if it's a grayscale picture

Those are the shapes matplotlib can deal with. Here is what my workflow looks like:

import numpy as np
import matplotlib.pyplot as plt

h, w = 50, 100

grey = np.random.randint(0, 256, (h, w, 1))
color = np.random.randint(0, 256, (h, w, 3)

With the same line, I need to be able to plot images that can either be color or greyscale.

plt.imshow(grey[:, :, 0 if grey.shape[-1] == 1 else :]
plt.imshow(color[:, :, 0 if color.shape[-1] == 1 else :]

But that's not correct python syntax.

Output:
Can't you just write

plt.imshow(grey[:,:,0] if grey.shape[-1] == 1 else grey)
plt.imshow(color[:,:,0] if color.shape[-1] == 1 else color)


Instruction: Need pytorch help doing 2D convolutions of N images with N kernels all at once

Input:
I have a torch tensor which is a stack of images. Let's say for kicks it is

im=th.arange(4*5*6,dtype=th.float32).view(4,5,6)

which is a 4x5x6 tensor, meaning four 5x6 images stacked vertically.

I want to convolve each layer with its own 2-D kernel so that

I_{out,j} = k_j*I_{in,j}, j=(1...4)

I can obviously do this with a for loop, but I'd like to take advantage of GPU acceleration and do all the convolutions at the same time. No matter what I try, I've only been able to use torch's conv2d or conv3d to produce a single output layer that is the sum of all the 2d convolutions. Or I can make 4 layers where each is the same sum of all the 2d convolutions. Here's a concrete example. Let's use im as defined above. Say that the kernel is defined by

k=th.zeros((4,3,3),dtype=th.float32)
n=-1
for i in range(2):
    for j in range(2):
        n+=1
        k[n,i,j]=1
        k[n,2,2]=1
print(k)

tensor([[[1., 0., 0.],
         [0., 0., 0.],
         [0., 0., 1.]],

        [[0., 1., 0.],
         [0., 0., 0.],
         [0., 0., 1.]],

        [[0., 0., 0.],
         [1., 0., 0.],
         [0., 0., 1.]],

        [[0., 0., 0.],
         [0., 1., 0.],
         [0., 0., 1.]]])

and from above, im is

tensor([[[  0.,   1.,   2.,   3.,   4.,   5.],
         [  6.,   7.,   8.,   9.,  10.,  11.],
         [ 12.,  13.,  14.,  15.,  16.,  17.],
         [ 18.,  19.,  20.,  21.,  22.,  23.],
         [ 24.,  25.,  26.,  27.,  28.,  29.]],

        [[ 30.,  31.,  32.,  33.,  34.,  35.],
         [ 36.,  37.,  38.,  39.,  40.,  41.],
         [ 42.,  43.,  44.,  45.,  46.,  47.],
         [ 48.,  49.,  50.,  51.,  52.,  53.],
         [ 54.,  55.,  56.,  57.,  58.,  59.]],

        [[ 60.,  61.,  62.,  63.,  64.,  65.],
         [ 66.,  67.,  68.,  69.,  70.,  71.],
         [ 72.,  73.,  74.,  75.,  76.,  77.],
         [ 78.,  79.,  80.,  81.,  82.,  83.],
         [ 84.,  85.,  86.,  87.,  88.,  89.]],

        [[ 90.,  91.,  92.,  93.,  94.,  95.],
         [ 96.,  97.,  98.,  99., 100., 101.],
         [102., 103., 104., 105., 106., 107.],
         [108., 109., 110., 111., 112., 113.],
         [114., 115., 116., 117., 118., 119.]]])

The right answer is easy if I do the for loop:

import torch.functional as F
for i in range(4):
    print(F.conv2d(im[i].expand(1,1,5,6),k[i].expand(1,1,3,3)))

tensor([[[[14., 16., 18., 20.],
          [26., 28., 30., 32.],
          [38., 40., 42., 44.]]]])
tensor([[[[ 75.,  77.,  79.,  81.],
          [ 87.,  89.,  91.,  93.],
          [ 99., 101., 103., 105.]]]])
tensor([[[[140., 142., 144., 146.],
          [152., 154., 156., 158.],
          [164., 166., 168., 170.]]]])
tensor([[[[201., 203., 205., 207.],
          [213., 215., 217., 219.],
          [225., 227., 229., 231.]]]])

As I noted earlier, the only thing I've been able to get is one sum of those four output images (or four copies of the same summed layer):

F.conv2d(im.expand(1,4,5,6),k.expand(1,4,3,3))

tensor([[[[430., 438., 446., 454.],
          [478., 486., 494., 502.],
          [526., 534., 542., 550.]]]])

I'm certain that what I want to do is possible, I just haven't been able to wrap my head around it yet. Does anyone have a solution to offer?

Output:
This is pretty straight forward if you use a grouped convolution.

From the nn.Conv2d documentation

  At groups=in_channels, each input channel is convolved with its own
  set of filters

Which is exactly what we want.

The shape of the weights argument to F.conv2d needs to be considered since it changes depending on the value of groups. The first dimension of weights should just be out_channels, which is 4 in this case. The second dimension according to F.conv2d docs should be in_channels / groups, which is 1. So we can perform the operation using

F.conv2d(im.unsqueeze(0), k.unsqueeze(1), groups=4).squeeze(0)

which produces a tensor of shape [4,3,4] with values

tensor([[[ 14.,  16.,  18.,  20.],
         [ 26.,  28.,  30.,  32.],
         [ 38.,  40.,  42.,  44.]],

        [[ 75.,  77.,  79.,  81.],
         [ 87.,  89.,  91.,  93.],
         [ 99., 101., 103., 105.]],

        [[140., 142., 144., 146.],
         [152., 154., 156., 158.],
         [164., 166., 168., 170.]],

        [[201., 203., 205., 207.],
         [213., 215., 217., 219.],
         [225., 227., 229., 231.]]])


Instruction: Python not finding torch

Input:
I want to use deepsaber.
For that I have to Install Torch.

I did this using pip install torch torchvision.

After running cd scripts/generation and ./script_generate.sh [path to song] python tells me:

Traceback (most recent call last):
  File "generate_stage1.py", line 16, in <module>
    from models import create_model
  File "/home/server/deepsaber/models/__init__.py", line 2, in <module>
    from .base_model import BaseModel
  File "/home/server/deepsaber/models/base_model.py", line 2, in <module>
    import torch
ModuleNotFoundError: No module named 'torch'
Traceback (most recent call last):
  File "generate_stage2.py", line 16, in <module>
    from models import create_model
  File "/home/server/deepsaber/models/__init__.py", line 2, in <module>
    from .base_model import BaseModel
  File "/home/server/deepsaber/models/base_model.py", line 2, in <module>
    import torch
ModuleNotFoundError: No module named 'torch'

using import torch works like a charm when using python

any ideas of what I can do?

The repos says:

  pytorch (installed as torch or via https://pytorch.org/get-started/locally/)

Output:
Most likely you installed torch using Python 2, while script_generate.sh uses Python 3 (see here):

# [...]
py=python3
# [...]

Try to run pip3 install torch torchvision or python3 -m pip install torch torchvision. Also, check if import torch works when using python3.


Instruction: how to #include in a .cpp file with Visual Studio?

Input:
Now I'm working with a project requiring using Pytorch C++ extension.
I've installed a Pytorch of version 1.4.0 in a python virtual environment:

activate crfasrnn
>>>import torch
>>>print(torch.__version__)
1.4.0

I'm using visual studio as a C++ compiler.
The code is like this:

#include<iostream>
#include<torch/extension.h>

int main()
{
    std::cout << "hello, world!" << std::endl;
    return 0;
}

However, the compiler tells me "can not open 'torch/extension.h'"
How can I solve this problem?

Output:
You need to get the appropriate LibTorch binaries from https://pytorch.org/ and then include the required binaries in the visual studio.
You can follow the detailed instructions given here.


Instruction: How can I interleave 5 PyTorch tensors?

Input:
I have 5 tensors of shape torch.Size([7, 20, 180])

I want to interleave them, one after the other along dim=1. So that my final shape will be torch.Size([7, 100, 180]).

Basically, I want the first element from the first tensor, then the first element from the second tensor, and so on.

Output:
If I understood correctly,

import torch
stacked = torch.stack(list_of_tensors, dim=2)
interleaved = torch.flatten(stacked, start_dim=1, end_dim=2)

interleaved is what you need apparently

(tested with pytorch 1.1.0)


Instruction: Pytorch: How to access tensor(values) by tensor(keys) in python dictionary

Input:
I have a dictionary with tensor keys and tensor values. I want to access the values by the keys.

from torch import tensor
x = {tensor(0): [tensor(1)], tensor(1): [tensor(0)]}
for i in x.keys():
  print(i, x[i])

Returns:

tensor(0) [tensor(1)]
tensor(1) [tensor(0)]

But when i try to access the values without looping through the keys,

try:
    print(x[tensor(0)])

except:
    print(Exception)
    print(x[0])

Throws Exception:

 KeyError                                   Traceback (most recent call last)
 <ipython-input-34-746d28dcd450> in <module>()
  6 try:
  ----> 7   print(x[tensor(0)])
  8

  KeyError: tensor(0)

  During handling of the above exception, another exception occurred:

  KeyError                                  Traceback (most recent call last)
  <ipython-input-34-746d28dcd450> in <module>()
  9 except:
  10   print(Exception)
  ---> 11   print(x[0])
  12 continue

  KeyError: 0

Output:
There's at least three issues here.

If x is a dictionary of tensor keys, then of course x[0] will not work.  0 is not a key of it.  Hence the inner KeyError that occured during the other exception.
Not actually relevant to your errors, but print(Exception) is almost surely not what you want.  It prints the class object (if that is the right term) of the class Exception.  You likely rather meant

except Exception as e:
  print(e)

or more specifically, except KeyError (otherwise it will just catch every kind of exception).
The real thing: you don't want to use a tensor as key in the first place.  It's a mutable type, compared by reference, not by value.  Every tensor(something) call will create a new object, hashing to a different value than the tensor(something) you specified as key.

Use the actual integers instead.


Instruction: How to use masked select for these kind of tensors?

Input:
Suppose, I got a tensor a and a tensor b

import torch
a = torch.tensor([[[ 0.8856,  0.1411, -0.1856, -0.1425],
    [-0.0971,  0.1251,  0.1608, -0.1302],
    [-0.0901,  0.3215,  0.1763, -0.0412]],
    [[ 0.8856,  0.1411, -0.1856, -0.1425],
    [-0.0971,  0.1251,  0.1608, -0.1302],
    [-0.0901,  0.3215,  0.1763, -0.0412]]])
b = torch.tensor([[0,
    2,
    1],
    [0,
    2,
    1]])

Now, I would like to select indices from tensor a, where the value of tensor b is not 0.

pred_masks = ( b != 0 )
c = torch.masked_select( a, (pred_masks == 1))

And of course, I get an expected error.

----> 1 c = torch.masked_select( a, (pred_masks == 1))

RuntimeError: The size of tensor a (4) must match the size of tensor b (3) at non-singleton dimension 2

This is caused by the nested list containing 4 items. However, it is required to select all the values of the nested list at index x in tensor a, corresponding to the index x in tensor b.

I will be grateful for any hint or answer.

Output:
I am not so sure what you want as the shape of the output c. Since your mask is of shape (2,3) and a is of shape (2,3,4) do you want as output a tensor of shape (n,4) where n is the number of elements that is true in the (2,3)-mask ?

If yes then I would suggest just using the mask as an index for the first two dimensions.

c = a[pred_masks,:]

Hope that helps a bit.


Instruction: IndexError: tensors used as indices must be long, byte or bool tensors

Input:
I am getting this error only during the testing phase, but I do not face any problem in the training and validation phase.

IndexError: tensors used as indices must be long, byte or bool tensors

I get this error for the last line in the given code snippet.

The code snippet looks like the one below,

NumClass = 10
mask = torch.zeros(batch_size, self.mem_dim, 4, 4)
ones = torch.ones(1, 4, 4)
NumRows = self.mem_dim
Elements = NumRows//NumClass
for i in range(batch_size):
    lab = torch.arange(Elements * label[i], Elements*(label[i]+1), 1)
    mask[i,lab] = ones

The "lab" is a tensor value and prints out the range in such a way,

tensor([6, 7, 8])
tensor([ 9, 10, 11])
tensor([21, 22, 23])

(Note*: the length of this lab tensor can be of length 'n' based on the value of ElementsPerClass)

Output: Yes it works when I provide a dtype=long to the label tensor and leaving rest of the tensors with default dtypes. thanks!


Instruction: Does BertForSequenceClassification classify on the CLS vector?

Input:
I'm using the Huggingface Transformer package and BERT with PyTorch. I'm trying to do 4-way sentiment classification and am using BertForSequenceClassification to build a model that eventually leads to a 4-way softmax at the end.

My understanding from reading the BERT paper is that the final dense vector for the input CLS token serves as a representation of the whole text string:

  The first token of every sequence is always a special classification token ([CLS]). The final hidden state corresponding to this token is used as the aggregate sequence representation for classification tasks.

So, does BertForSequenceClassification actually train and use this vector to perform the final classification?

The reason I ask is because when I print(model), it is not obvious to me that the CLS vector is being used.

model = BertForSequenceClassification.from_pretrained(
    model_config,
    num_labels=num_labels,
    output_attentions=False,
    output_hidden_states=False
)

print(model)

Here is the bottom of the output:

        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (dropout): Dropout(p=0.1, inplace=False)
  (classifier): Linear(in_features=768, out_features=4, bias=True)

I see that there is a pooling layer BertPooler leading to a Dropout leading to a Linear which presumably performs the final 4-way softmax. However, the use of the BertPooler is not clear to me. Is it operating on only the hidden state of CLS, or is it doing some kind of pooling over hidden states of all the input tokens?

Thanks for any help.

Output:
The short answer: Yes, you are correct. Indeed, they use the CLS token (and only that) for BertForSequenceClassification.

Looking at the implementation of the BertPooler reveals that it is using the first hidden state, which corresponds to the [CLS] token.
I briefly checked one other model (RoBERTa) to see whether this is consistent across models. Here, too,  classification only takes place based on the [CLS] token, albeit less obvious (check lines 539-542 here).


Instruction: Cuda version for cat in Pytorch?

Input:
I am trying to build a CNN architecture for Word Embedding. I am trying to concatenate two tensors using torch.cat but its throwing this error:

     22         print(z1.size())
---> 23         zcat = torch.cat(lz, dim = 2)
     24         print("zcat",zcat.size())
     25         zcat2=zcat.reshape([batch_size, 1, 100, 3000])

RuntimeError: Expected object of backend CUDA but got backend CPU for sequence element 1 in sequence argument at position #1 'tensors'

Attaching the architecture for reference as well:

    def __init__(self,vocab_size,embedding_dm,pad_idx):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size,embedding_dim,padding_idx = pad_idx)
        self.convs = nn.ModuleList([nn.Conv2d(in_channels = 1,out_channels = 50,kernel_size = (1,fs)) for fs in (3,4,5)])
        self.conv2 = nn.Conv2d(in_channels = 50,out_channels = 100,kernel_size = (1,2))
        self.fc1 = nn.Linear(100000,150) #Change this
        self.fc2 = nn.Linear(150,1)
        self.dropout = nn.Dropout(0.5)
    def forward(self,text):
        print("text",text.size())
        embedded = self.embedding(text.T)
        embedded = embedded.permute(0, 2,1)
        print("embedded",embedded.size())
        x=embedded.size(2)
        y=3000-x
        print(y,"hello")
        batch_size=embedded.size(0)
        z=np.zeros((batch_size,100,y))
        z1=torch.from_numpy(z).float()
        lz=[embedded,z1]
        print(z1.size())
        zcat = torch.cat(lz, dim = 2)
        print("zcat",zcat.size())
        zcat2=zcat.reshape([batch_size, 1, 100, 3000])
        print("zcat2",zcat2.size())
#         embedded = embedded.reshape([embedded.shape[0],1,])
        print(embedded.size(),"embedding")
        conved = [F.relu(conv(embedded)) for conv in self.convs]
        pooled = [F.max_pool2d(conv,(1,2)) for conv in conved]
        print("Pool")
        for pl in pooled:
            print(pl.size())
        cat = torch.cat(pooled,dim = 3)
        print("cat",cat.size())
        conved2 = F.relu(self.conv2(cat))
        print("conved2",conved2.size())
        pooled2 = F.max_pool2d(conved2,(1,2))
        print(pooled2.size(),"pooled2")
        return 0
#         return pooled2

Am I doing something wrong? Help appreciated. Thanks!

Output:
Got it.
Just create a tensor using :
- torch.zeros(batch_size,100,y,dtype = embedded.dtype,device = embedded.device)


Instruction: How can I get the mean of 5 tensors along an axis?

Input:
I have 5 tensors of shape torch.Size([7, 20, 180])

I want to get the mean of each along dim=1 so that my final shape will be torch.Size([7, 20, 180]).

Basically, I want the first element from the first tensor, then the first element from the second tensor, and so on to be averaged.

Output:
You did not mention how these 5 tensors are stored, but let's assume they are in a list. Here's a way to do it:

import torch

x = [torch.rand((7, 20, 180)) for _ in range(5)]
y = torch.stack(x).mean(dim=0)

print(y.shape)
# >>> torch.Size([7, 20, 180])

I'm also assuming you said dim=1 as if PyTorch was 1-based indexing, which it is not.

I see you are asking many questions recently, which is not a problem. I've said this once, but again: always try to provide a Minimal, Reproducible Example. It is always good to show some effort as well. Have you tried anything before asking?


Instruction: How can I remove elements across a dimension that are all zero with PyTorch?

Input:
I have a tensor:

tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1.,
         1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]],
       device='cuda:0', grad_fn=<AsStridedBackward>)
torch.Size([3, 176])

You see that the first and the third have all zeroes. I'd like to remove it and change this to have size of [1, 176].

If I do print((mask != 0).nonzero()), then I see:

tensor([[  0, 136],
        [  0, 137],
        [  0, 138],
        [  0, 139],
        [  0, 140],
        [  0, 141],
        [  0, 142],
        [  0, 143],
        [  0, 144],
        [  0, 145],
        [  0, 146],
        [  0, 147],
        [  0, 148],
        [  0, 149],
        [  0, 150],
        [  0, 151],
        [  0, 152],
        [  0, 153],
        [  0, 154],
        [  0, 155],
        [  0, 156],
        [  0, 157],
        [  0, 158],
        [  0, 159]], device='cuda:0')

So I feel like that's a part of it, but how do I further reduce the tensor?

Output:
# let x be some tensor with nonzero somewhere
x = torch.zeros((2, 5))
x[1, 2] = 1

nonZeroRows = torch.abs(x).sum(dim=1) > 0
x = x[nonZeroRows]


Instruction: How can I trim a tensor based on a mask with PyTorch?

Input:
I have a tensor inp, which has a size of: torch.Size([4, 122, 161]).

I also have a mask with a size of: torch.Size([4, 122]).

Each element in my mask looks something like:

tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
        1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       device='cuda:0', grad_fn=<SelectBackward>)

So I want to trim inp to be reduced along the dimension=1 to only exist where the mask has 1. In the case shown, there are 23 1s, so I want the size of inp to be: torch.Size([4, 23, 161])

Output:
I think advanced indexing would work. (I assume every mask has equally 23 1s)

inp_trimmed = inp[mask.type(torch.bool)].reshape(4,23,161)


Instruction: How can I element-wise multiply tensors with different dimensions?

Input:
I have a tensor expanded_mask, which has a size of torch.Size([1, 208]) and another one inputs which has a size of torch.Size([1, 208, 161]).

I want to elementwise multiply expanded_mask and input such that all 161 elements of the third dimension are multiplied with the 208 elements of expanded_mask.

As per jodag's answer, I tried:

masked_inputs = expanded_mask.unsqueeze(2) * inputs

inputs is:

tensor([1.8851e-02, 4.4921e-02, 7.5260e-02, 3.8994e-02, 3.5651e-02, 3.0457e-02,
        1.2933e-02, 2.5496e-02, 2.3260e-04, 2.4903e-03, 6.5678e-03, 1.0501e-02,
        1.2387e-02, 1.9434e-03, 1.0831e-03, 6.5691e-03, 5.3792e-03, 9.1925e-03,
        1.8146e-03, 4.9215e-03, 1.4623e-03, 9.4454e-03, 1.0504e-03, 3.3749e-03,
        2.1361e-03, 8.0782e-03, 1.7916e-03, 1.1577e-03, 1.1246e-04, 2.2520e-03,
        2.2255e-03, 2.1072e-03, 9.8782e-03, 2.2909e-03, 2.9957e-03, 5.8540e-03,
        1.1067e-02, 9.0582e-03, 5.6360e-03, 6.3841e-03, 5.9298e-03, 1.9501e-04,
        2.7967e-03, 3.5786e-03, 9.2363e-03, 8.3934e-03, 8.8185e-04, 5.4591e-03,
        2.2451e-04, 2.2307e-03, 2.4871e-03, 3.6736e-03, 1.3842e-04, 2.7455e-03,
        6.2199e-03, 1.1924e-02, 9.5953e-03, 1.6939e-03, 4.1919e-04, 9.3509e-05,
        1.8351e-03, 6.3350e-04, 1.1076e-03, 1.5472e-03, 1.2104e-03, 3.1803e-04,
        8.6507e-04, 3.0083e-03, 2.8435e-03, 1.6740e-03, 8.1023e-05, 7.5767e-04,
        9.1442e-04, 2.0204e-03, 1.3987e-03, 3.7729e-03, 5.2012e-04, 2.0367e-03,
        1.5177e-03, 1.6948e-03, 9.5833e-04, 1.2050e-03, 1.8356e-03, 9.4503e-04,
        4.8612e-04, 1.6844e-04, 1.2222e-04, 1.7526e-03, 2.6397e-04, 1.3026e-03,
        1.0704e-03, 3.6407e-04, 1.3135e-03, 2.6665e-03, 1.8639e-03, 3.0385e-05,
        1.0212e-03, 7.6236e-04, 1.7878e-03, 2.4298e-03, 7.2158e-05, 1.2488e-03,
        2.1347e-03, 3.9256e-03, 3.1436e-03, 3.1648e-03, 3.4657e-03, 1.3746e-03,
        1.6927e-03, 1.0794e-03, 8.8152e-04, 1.1757e-04, 3.2254e-04, 4.1866e-04,
        9.2787e-04, 2.0020e-03, 1.4813e-03, 1.1912e-03, 2.4577e-03, 2.2247e-03,
        1.7862e-03, 1.7460e-03, 1.4388e-03, 4.3175e-04, 6.7808e-04, 2.6875e-04,
        3.6475e-04, 8.7643e-04, 3.6790e-04, 2.1274e-04, 6.3725e-04, 2.0949e-03,
        2.4069e-03, 1.7348e-03, 1.0026e-03, 1.2451e-03, 4.7888e-04, 5.9790e-04,
        1.4343e-03, 4.0900e-03, 1.0176e-03, 5.5178e-04, 2.0624e-03, 1.2878e-03,
        6.9607e-04, 4.3259e-04, 1.8573e-03, 7.5521e-04, 5.2949e-04, 3.4758e-04,
        4.7898e-04, 7.5599e-04, 6.0631e-04, 1.7585e-03, 1.8156e-03, 3.2421e-04,
        8.9446e-04, 7.2131e-04, 6.2817e-04, 1.0827e-03, 2.0211e-03],
       device='cuda:0')

expanded_mask is:

tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
         1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], device='cuda:0',
       grad_fn=<AsStridedBackward>)

then masked_inputs is:

tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
       device='cuda:0', grad_fn=<SelectBackward>)

Looks like the 1's isn't being multiplied through.

Output:
Another way of using broadcasting:

import torch

mask = torch.tensor([[1, 0, 1]])
inputs = torch.randn(1, 3, 2)
masked = inputs * mask[..., None]

print(mask)
print(inputs)
print(masked)

result:

tensor([[1, 0, 1]])
tensor([[[ 2.2820,  2.7476],
         [-0.1738, -0.5703],
         [ 0.7077, -0.6384]]])
tensor([[[ 2.2820,  2.7476],
         [-0.0000, -0.0000],
         [ 0.7077, -0.6384]]])

The ellipsis operator denotes all dimensions, then None adds a dimension at the end.


Instruction: How to load BertforSequenceClassification models weights into BertforTokenClassification model?

Input:
Initially, I have a fine-tuned BERT base cased model using a text classification dataset and I have used BertforSequenceClassification class for this.

from transformers import BertForSequenceClassification, AdamW, BertConfig

# Load BertForSequenceClassification, the pretrained BERT model with a single
# linear classification layer on top.
model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased", # Use the 12-layer BERT model, with an uncased vocab.
    num_labels = 2, # The number of output labels--2 for binary classification.
                    # You can increase this for multi-class tasks.
    output_attentions = False, # Whether the model returns attentions weights.
    output_hidden_states = False, # Whether the model returns all hidden-states.
)

Now I want to use this fine-tuned BERT model weights for Named Entity Recognition and I have to use BertforTokenClassification class for this. I'm unable to figure out how to load the fine-tuned BERT model weights into the new model created using BertforTokenClassification.

Thanks in advance.......................

Output:
You can get weights from the bert inside the first model and load into the bert inside the second:

new_model = BertForTokenClassification(config=config)
new_model.bert.load_state_dict(model.bert.state_dict())


Instruction: Torchvision.transforms implementation of Flatten()

Input:
I have grayscale images, but I need transform it to a dataset of 1d vectors
How can I do this? I could not find a suitable method in transforms:
train_dataset = torchvision.datasets.ImageFolder(root='./data',train=True, transform=transforms.ToTensor())
test_dataset = torchvision.datasets.ImageFolder(root='./data',train=False, transform=transforms.ToTensor())

train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=4, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=4, shuffle=False)

Output:
Here's how you can do it using Lambda

import torch
from torchvision.datasets import MNIST
import torchvision.transforms as T

# without flatten
dataset = MNIST(root='.', download=True, transform=T.ToTensor())
print(dataset[0][0].shape)
# >>> torch.Size([1, 28, 28])

# with flatten (using Lambda, but you can do it in many other ways)
dataset_flatten = MNIST(root='.', download=True, transform=T.Compose([T.ToTensor(), T.Lambda(lambda x: torch.flatten(x))]))
print(dataset_flatten[0][0].shape)
# >>> torch.Size([784])


Instruction: How can I set precision when printing a PyTorch tensor with integers?

Input:
I have:

mask = mask_model(input_spectrogram)
mask = torch.round(mask).float()
torch.set_printoptions(precision=4)
print(mask.size(), input_spectrogram.size(), masked_input.size())
print(mask[0][-40:], mask[0].size())

This prints:

tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
        0., 0., 0., 0.], grad_fn=<SliceBackward>) torch.Size([400])

But I want it to print 1.0000 instead of 1.. Why won't my set_precision do it? Even when I converted to float()?

Output: Unfortunately, this is simply how PyTorch displays tensor values. Your code works fine, if you do print(mask * 1.1), you can see that PyTorch does indeed print out 4 decimal values when the tensor values can no longer be represented as integers.


Instruction: How to deploy Pytorch in Python via a REST API with Flask?

Input:
I am working on AWS Sagemaker and my goal is to follow this tutorial from Pytorch's official documentation.

The original predict function from the tutorial above is the following:

@app.route('/predict', methods=['POST'])
def predict():
    if request.method == 'POST':
        file = request.files['file']
        img_bytes = file.read()
        class_id, class_name = get_prediction(image_bytes=img_bytes)
        return jsonify({'class_id': class_id, 'class_name': class_name})

I was getting this error, so I added 'GET' as a method as mentioned in here. I also simplified my example to its minimal expression:

from flask import Flask, jsonify, request

app = Flask(__name__)

@app.route('/predict', methods=['GET','POST'])
def predict():
    if request.method == 'POST':
        return jsonify({'class_name': 'cat'})
    return 'OK'

if __name__ == '__main__':
    app.run()

I perform requests with the following code:

import requests

resp = requests.post("https://catdogclassifier.notebook.eu-west-1.sagemaker.aws/proxy/5000/predict",
                     files={"file": open('/home/ec2-user/SageMaker/cat.jpg', 'rb')})

resp is <Response [200]> but resp.json() returns JSONDecodeError: Expecting value: line 1 column 1 (char 0) Finally, resp.url points me to a page saying 'OK'.

Moreover, this is the output of resp.content

<!DOCTYPE HTML>

What am I missing?

Output:
Looks like the content of your resp is HTML as opposed to JSON; this is likely a consequence of how the Jupyter server proxy endpoint you're attempting to POST to (https://catdogclassifier.notebook.eu-west-1.sagemaker.aws/proxy/5000/predict) is configured.

It looks like you're using a SageMaker notebook instance, so you might not have much control over this configuration. A workaround could be to instead deploy your Flask server as a SageMaker endpoint running outside JupyterLab, instead of directly on a notebook instance.

If you want to prototype using only a notebook instance, you can alternately just bypass the proxy entirely and simply call your Flask route relative to localhost from another notebook tab while the Flask server runs in your main notebook tab:

import requests

resp = requests.post("https://localhost:5000/predict",
                     files={"file": open('/home/ec2-user/SageMaker/cat.jpg', 'rb')})


Instruction: How can I select all values of a multi dimensional tensor based on a mask with PyTorch?

Input:
I have a tensor pred, which has a .size() of [4, 53, 161]. I have another tensor mask, which has a .size() of [4, 53].

The mask is only 0's and 1's. What I want to do is get the values of pred where mask has a value of 1. You'll notice that pred has one more dimension than mask. How can I accomplish this?

Output:
mask = mask.unsqueeze(2)
new_pred = pred * mask

That will add extra dimension. It will be now [4, 53, 1]. Rest will be taken care of the broadcasting. (if you do some operation)

Suppose you have a tensor of shape[4, 53, 164] and now you want to reduce it to [4, 53] then you can apply arithmetic operations like this new_pred.mean(2).


Instruction: How can I change part of a PyTorch tensor based on the values of another tensor?

Input:
This question may not be clear, so please ask for clarification in the comments and I will expand.

I have the following tensors of the following shape:

mask.size() == torch.Size([1, 400])
clean_input_spectrogram.size() == torch.Size([1, 400, 161])
output.size() == torch.Size([1, 400, 161])

mask is comprised only of 0 and 1. Since it's a mask, I want to set the elements of output equal to clean_input_spectrogram where that relevant mask value is 1.

How would I do that?

Output:
You can do something like this, where:

m is your mask;
x is your spectogram;
o is your output;

import torch
torch.manual_seed(2020)

m = torch.tensor([[0, 1, 0]]).to(torch.int32)
x = torch.rand((1, 3, 2))
o = torch.rand((1, 3, 2))

print(o)
# tensor([[[0.5899, 0.8105],
#          [0.2512, 0.6307],
#          [0.5403, 0.8033]]])
print(x)
# tensor([[[0.4869, 0.1052],
#          [0.5883, 0.1161],
#          [0.4949, 0.2824]]])

o[:, m[0].to(torch.bool), :] = x[:, m[0].to(torch.bool), :]
# or
# o[:, m[0] == 1, :] = x[:, m[0] == 1, :]

print(o)
# tensor([[[0.5899, 0.8105],
#          [0.5883, 0.1161],
#          [0.5403, 0.8033]]])


Instruction: Create a specific tensor from another tensor

Input:
q_pred = self.Q.forward(states) gives me the following output :

tensor([[-4.4713e-02,  4.2878e-03],
        [-2.2801e-01,  2.2295e-01],
        [-9.8098e-03, -1.0766e-01],
        [-1.4654e-01,  1.2742e-01],
        [-1.6224e-01,  1.6565e-01],
        [-3.6515e-02,  3.1022e-02],
        [-4.5094e-02,  1.4848e-02],
        [-1.4157e-01,  1.3974e-01],
        [-3.0593e-03, -4.2342e-02],
        [-4.1689e-02,  2.9376e-02],
        [-9.3629e-02,  1.0297e-01],
        [-5.2163e-04, -7.4799e-02],
        [-2.8944e-02, -1.2417e-01]], grad_fn=<AddmmBackward>)

and actions gives me the the following output

tensor([1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1], dtype=torch.int32)

From the two above output, how can I can the following result :

tensor([4.2878e-03, -2.2801e-01, -1.0766e-01, 1.2742e-01, -1.6224e-01,  3.1022e-02, 1.4848e-02,
        -1.4157e-01, -4.2342e-02, -4.1689e-02, -9.3629e-02, -7.4799e-02,, -1.2417e-01], grad_fn=<AddmmBackward>)

?

UPDATE

Here is how self.Q is implemented :

class LinearDeepQNetwork(nn.Module):
    def __init__(self, lr, n_actions, input_dims):
        super(LinearDeepQNetwork, self).__init__()

        self.fc1 = nn.Linear(*input_dims, 128)
        self.fc2 = nn.Linear(128, n_actions)

        self.optimizer = optim.Adam(self.parameters(), lr=lr)
        self.loss  = nn.MSELoss()
        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')
        self.to(self.device)

    def forward(self, state):
        layer1 = F.relu(self.fc1(state))
        actions = self.fc2(layer1)

        return actions

Output:
you can use:

q_pred[torch.arange(q_pred.size(0)), actions.type(torch.LongTensor)]