Instruction: How can I use the LBFGS optimizer with pytorch ignite?

Input:
I started using Ignite recently and i found it very interesting.
I would like to train a model using as an optimizer the LBFGS algorithm from the torch.optim module.
This is my code:
from ignite.engine import Events, Engine, create_supervised_trainer, create_supervised_evaluator
from ignite.metrics import RootMeanSquaredError, Loss
from ignite.handlers import EarlyStopping

 D_in, H, D_out = 5, 10, 1
 model = simpleNN(D_in, H, D_out) # a simple MLP with 1 Hidden Layer
 model.double()
 train_loader, val_loader = get_data_loaders(i)

 optimizer = torch.optim.LBFGS(model.parameters(), lr=1)
 loss_func = torch.nn.MSELoss()

 #Ignite
 trainer = create_supervised_trainer(model, optimizer, loss_func)
 evaluator = create_supervised_evaluator(model, metrics={'RMSE': RootMeanSquaredError(),'LOSS': Loss(loss_func)})

 @trainer.on(Events.ITERATION_COMPLETED)
 def log_training_loss(engine):
     print("Epoch[{}] Loss: {:.5f}".format(engine.state.epoch, len(train_loader), engine.state.output))

def score_function(engine):
    val_loss = engine.state.metrics['RMSE']
    print("VAL_LOSS: {:.5f}".format(val_loss))
    return -val_loss

handler = EarlyStopping(patience=10, score_function=score_function, trainer=trainer)
evaluator.add_event_handler(Events.COMPLETED, handler)

trainer.run(train_loader, max_epochs=100)

And the error that raises is:
TypeError: step() missing 1 required positional argument: 'closure'
I know that is required to define a closure for the implementation of LBFGS, so my question is how can I do it using ignite? or is there another approach for doing this?

Output:
The way to do it is like this:
    from ignite.engine import Engine

    model = ...
    optimizer = torch.optim.LBFGS(model.parameters(), lr=1)
    criterion =

    def update_fn(engine, batch):
        model.train()
        x, y = batch
        # pass to device if needed as here: https://github.com/pytorch/ignite/blob/40d815930d7801b21acfecfa21cd2641a5a50249/ignite/engine/__init__.py#L45
        def closure():
            y_pred = model(x)
            loss = criterion(y_pred, y)
            optimizer.zero_grad()
            loss.backward()
            return loss

        optimizer.step(closure)

    trainer = Engine(update_fn)

    # everything else is the same

Source


Instruction: Can't replace classifier on Densenet121 in pytorch

Input:
I am trying to do some transfer learning using this github DenseNet121 model (https://github.com/gaetandi/cheXpert.git). I'm running into issues resizing the classification layer from 14 to 2 outputs.

Relevant part of the github code is:

class DenseNet121(nn.Module):
    """Model modified.
    The architecture of our model is the same as standard DenseNet121
    except the classifier layer which has an additional sigmoid function.
    """
    def __init__(self, out_size):
        super(DenseNet121, self).__init__()
        self.densenet121 = torchvision.models.densenet121(pretrained=True)
        num_ftrs = self.densenet121.classifier.in_features
        self.densenet121.classifier = nn.Sequential(
            nn.Linear(num_ftrs, out_size),
            nn.Sigmoid()
        )
def forward(self, x):
    x = self.densenet121(x)
    return x

I load and init with:

# initialize and load the model
model = DenseNet121(nnClassCount).cuda()
model = torch.nn.DataParallel(model).cuda()
modeldict = torch.load("model_ones_3epoch_densenet.tar")
model.load_state_dict(modeldict['state_dict'])

It looks like DenseNet doesn't split layers up into children so model = nn.Sequential(*list(modelRes.children())[:-1]) won't work.

model.classifier = nn.Linear(1024, 2) seems to work on default DenseNets, but with the modified classifier (additional sigmoid function) here it ends up just adding an additional classifier layer without replacing the original.

I've tried

model.classifier = nn.Sequential(
    nn.Linear(1024, dset_classes_number),
    nn.Sigmoid()
)

But am having the same added instead of replaced classifier issue:

...
      )
      (classifier): Sequential(
        (0): Linear(in_features=1024, out_features=14, bias=True)
        (1): Sigmoid()
      )
    )
  )
  (classifier): Sequential(
    (0): Linear(in_features=1024, out_features=2, bias=True)
    (1): Sigmoid()
  )
)

Output:
If you want to replace the classifier inside densenet121 that is a member of your model you need to assign

model.densenet121.classifier = nn.Sequential(...)


Instruction: Optimize Custom Parameters in Pytorch

Input:
I'm trying to create some custom parameters to optimize and came across this helpful link here. However, I'm a bit confused as to why this code works.

Here is the code with some slight modifications from the original post to clearly see the behavior of the optimization.

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class Mask(nn.Module):
    def __init__(self):
        super(Mask, self).__init__()
        self.weight = torch.nn.Parameter(data=torch.Tensor(5, 5), requires_grad=True)

        self.weight.data.uniform_(-1, 1)

    def forward(self, x):
        masked_wt = self.weight.mul(1)
        return masked_wt

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.mask = Mask()

    def forward(self,x):
        x = self.mask(x)
        return x

model = Model()
indata = torch.ones(5,5)
optimizer = torch.optim.Adam(model.parameters(),lr=0.001)

while True:
    x = model(indata)
    optimizer.zero_grad()
    loss = F.l1_loss(indata,x)
    loss.backward()
    optimizer.step()

    print(model.mask.weight)

So two questions:

In the forward function of the Mask class, why do I need to do self.weight.mul(1) and why doesn't x get used?
x = model(indata) clearly does per element multiplication between the 5x5 matrix indata and the weight, but how does this happen if we didn't use x in the forward function of Mask?

Output:
You are right, Mask.forward discard x completely. However, the output of your model "sees" indata when computing the loss.
What you are actually teaching your model is to have mask.weight == indata.


Instruction: What defines legitimate input dimensions of a pytorch nn?

Input:
From the tutorial / example github, the network to be trained on the MNIST dataset is defined as:

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4*4*50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4*4*50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

The image sizes this network is trained with are 28x28 pixels.

Testing images of different sizes creates errors. To be precise: I tested images of size 27x27 and 32x32 for whom it does break with an error; an input size of 29x29 does create none.

Where is the size of 28x28 actually defined? What formula can I use to discern parameters for different input sizes for different tasks? Can one use images of different sizes as input? How?

Output:
You can backtrack the size of the input image from the first fully connected layer i.e. fc1 = Linear(4*4*50, 500). The input to the fc1 is 50x4x4 (CxHxW) (here 50 is the channel dimension as evident from the previous conv2 layer). Thus, the output of conv2 (before max-pooling operation) is 50x8x8 as you're performing pooling using 2x2 - max_pool2d(x, 2, 2).

torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, ...)

Now, you can get the input size of image before the convolution operation using the formula (W-F+2P)/S + 1 = output size. Here, W is input size, F is filter/kernel size, and P is padding used, and S is the stride. Thus, (W-5+2*0)/1+1=8 => W=12.

Hence, the input to conv2 is 20x12x12.

In the same way, we can continue the process as follows:

Output of conv1 (i.e. before max-pooling): 20x24x24
Input to conv1: 1x28x28. ((W-5+2*0)/1+1=24 => W=28)

Hence, the input image size is 1x28x28.

The error is because the fully connected layers expect input of fixed size and that defines your network. In order to pass variable sized input, you may use need to transform the input to size that your network (fc layers) expects using transformations such as cropping.

Also, there are networks that can take variable size input e.g. Fully Convolutional Networks (FCN), which doesn't contain fc layers, but only conv layers. You may also read about Spatial Pyramid Pooling (used in a network called DeepLab for semantic segmentation). Read more here.


Instruction: Deep learning with Pytorch: understanding the neural network example

Input:
I'm reading the Pytorch documentation and I have a couple of questions about the neural network that is introduced. The documentation defines the following network:

import torch
import torch.nn as nn
import torch.nn.functional as F

    class Net(nn.Module):

        def __init__(self):
            super(Net, self).__init__()
            # 1 input image channel, 6 output channels, 3x3 square convolution
            # kernel
            self.conv1 = nn.Conv2d(1, 6, 3)
            self.conv2 = nn.Conv2d(6, 16, 3)
            # an affine operation: y = Wx + b
            self.fc1 = nn.Linear(16 * 6 * 6, 120)  # 6*6 from image dimension
            self.fc2 = nn.Linear(120, 84)
            self.fc3 = nn.Linear(84, 10)

        def forward(self, x):
            # Max pooling over a (2, 2) window
            x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
            # If the size is a square you can only specify a single number
            x = F.max_pool2d(F.relu(self.conv2(x)), 2)
            x = x.view(-1, self.num_flat_features(x))
            x = F.relu(self.fc1(x))
            x = F.relu(self.fc2(x))
            x = self.fc3(x)
            return x

        def num_flat_features(self, x):
            size = x.size()[1:]  # all dimensions except the batch dimension
            num_features = 1
            for s in size:
                num_features *= s
            return num_features

Later on, the following statement is made:

Let try a random 32x32 input. Note: expected input size of this net (LeNet) is 32x32. To use this net on MNIST dataset, please resize the images from the dataset to 32x32.

Question 1: Why do the images need to be 32x32 (where I assume that this means 32 pixels by 32)?

The first convolution applies six kernel to an image, with every kernel being 3x3. This means that if the input channel is 32x32, the six output channels all have dimensions 30x30 (the 3x3 kernel grid makes you lose 2 pixels in width and height). The second convolution applies more kernels so that there now are sixteen output channels of dimensions 28x28 (again the 3x3 kernel grid makes you lose 2 pixels in width and height). Now I would expect 16x28x28 nodes in the next layer, since every one of the sixteen output channels has 28x28 pixels. Somehow, this is incorrect, and the next layer contains 16x6x6 nodes. Why is this true?

Question 2: The second convolution layer goes from six input channels to sixteen output channels. How is this done?

In the first convolution layer we go from one input channel to six input channels, which makes sense to me. You can just apply six kernels to the single input channel to arrive at six output channels. Going from six input channels to sixteen output channels does not make as much sense to me. How are the different kernels applied? Do you apply two kernels to the first five input channels to arrive at ten output channels, and apply six kernels to the last input channel, so that the total comes to sixteen output channels? Or does the neural network learn itself to use x kernels and apply them to the input channels that it finds most suitable?

Output:
I can now answer these questions my self.

Question 1: To see why you need a 32x32 image for this neural network to work consider the following:

Layer 1:
First, convolution is applied with a 3x3 kernel. Since the image has dimensions 32x32, this will result in a grid of 30x30. Next, max pooling is applied to the grid, with a 2x2 kernel and stride of 2 resulting in a grid that has dimensions 15x15.

Layer 2:
First, convolution is applied with a 3x3 kernel to the 15x15 grid, resulting in a 13x13 grid. Next, max pooling is applied with a 2x2 kernel and stride of 2 resulting in a grid that has dimensions 6x6. We get a 6x6 grid and not a 7x7 grid because by default the floor function is used and not the ceil function.

Since the convolution in layer 2 has sixteen output channels, the first linear layer needs 16x6x6 nodes! We see that the required input is indeed a 32x32 image.

Question 2: Every output channel is created by applying six different kernels to each input channel and summing the results. This is explained in the documentation.


Instruction: epsilon parameter in Adam opitmizer

Input:
Using pyTorch and tensorflow (TF), I was wandering how the Adam optimizer is implemented for curiosity. And I do not know if I am wrong or not but it seems to me that the two implementations differ and the pyTorch one is the original one from https://arxiv.org/pdf/1412.6980.pdf.

My problem comes from the eps-parameter. Using the TF implentation seems to lead to a time-and-b2 dependance of this parameter, namely

q(t+1) = q(t) - \gamma * sqrt[(1-b2^t)]/(1-b1^t) * m(t)/[sqrt[v(t)]+eps]

which in the original algorithm notation can be reformulate as

 q(t+1) = q(t) - \gamma * mhat(t)/[sqrt[vhat(t)]+ eps/sqrt[(1-b2^t)]]

and this point out the variation of the eps-parameter which is not the case neither in the original algorithm neither in the pyTorch implementation.

Am I wrong? or it is well known?
Thanks for your help.

Output:
Indeed, you can check this in the docs for the TF Adam optimizer. To quote the relevant part:

  The default value of 1e-8 for epsilon might not be a good default in general. For example, when training an Inception network on ImageNet a current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the formulation just before Section 2.1 of the Kingma and Ba paper rather than the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon hat" in the paper.

If you check the "the formulation just before section 2.1" in the paper, they actually include the time dependence in alpha, resulting in a time-dependent "step size" alpha_t but a fixed epsilon. Note that at the end of the day, this is just rewriting/interpreting parameters in a slightly different fashion and doesn't change the actual workings of the algorithm. But you will need to be aware that choosing the same epsilon in the PyTorch and TF implementations will apparently not lead to the same results...


Instruction: Convert CUDA tensor to NumPy

Input:
First of all, I tried those solutions: 1, 2, 3, and 4, but did not work for me.

After training and testing the neural network, I am trying to show some examples to verify my work. I named the method predict which I pass the image to it to predict for which class it belongs:

def predict(model, image_path, topk=5):
''' Predict the class (or classes) of an image using a trained deep learning model.
'''

output = process_image(image_path)
output.unsqueeze_(0)
output = output.cuda().float()

model.eval()

with torch.no_grad():
    score = model(output)
    prob, idxs = torch.topk(score, topk)

    # Convert indices to classes
    idxs = np.array(idxs)
    idx_to_class = {val:key for key, val in model.class_to_idx.items()}
    classes = [idx_to_class[idx] for idx in idxs[0]]

    # Map the class name with collected topk classes
    names = []
    for cls in classes:
        names.append(cat_to_name[str(cls)])

    return prob, names

Then there is the final step which displays the final result based on the training of the neural network and done like this:

# TODO: Display an image along with the top 5 classes
x_pos, y_pos = predict(model, img_pil, topk=5)

ax_img = imshow(output)
ax_img.set_title(y_pos[0])

plt.figure(figsize=(4,4))
plt.barh(range(len(y_pos)), np.exp(x_pos[0]))
plt.yticks(range(len(y_pos)), y_pos)

plt.show()

The error is:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-45-e3f9951e9804> in <module>()
----> 1 x_pos, y_pos = predict(model, img_pil, topk=5)
      2
      3 ax_img = imshow(output)
      4 ax_img.set_title(y_pos[0])
      5

1 frames
<ipython-input-44-d77500f31561> in predict(model, image_path, topk)
     14
     15         # Convert indices to classes
---> 16         idxs = np.array(idxs)
     17         idx_to_class = {val:key for key, val in model.class_to_idx.items()}
     18         classes = [idx_to_class[idx] for idx in idxs[0]]

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in __array__(self, dtype)
    456     def __array__(self, dtype=None):
    457         if dtype is None:
--> 458             return self.numpy()
    459         else:
    460             return self.numpy().astype(dtype, copy=False)

TypeError: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

How do I solve this?

I tried to change idx to idxs = idxs.cpu().numpy() and the error is:

TypeError                                 Traceback (most recent call last)
<ipython-input-62-e3f9951e9804> in <module>()
      5
      6 plt.figure(figsize=(4,4))
----> 7 plt.barh(range(len(y_pos)), np.exp(x_pos[0]))
      8 plt.yticks(range(len(y_pos)), y_pos)
      9

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in __array__(self, dtype)
    456     def __array__(self, dtype=None):
    457         if dtype is None:
--> 458             return self.numpy()
    459         else:
    460             return self.numpy().astype(dtype, copy=False)

TypeError: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

Output:
Try to change

idxs = np.array(idxs)

to

idxs = idxs.cpu().numpy()

And change

plt.barh(range(len(y_pos)), np.exp(x_pos[0]))

to

plt.barh(range(len(y_pos)), np.exp(x_pos[0].cpu().numpy()))


Instruction: Pytorch: choosing columns from a 3d tensor, according to indices tensor

Input:
I have a 3D tensor M of dimensions [BxLxD] and a 1D tensor idx of dimensions [B,1] that contains column indices in the range (0, L-1). I want to create a 2D tensor N of dimensions [BxD] such that N[i,j] = M[i, idx[i], j]. How can this be done efficiently?

Example:

B,L,D = 2,4,2

M = torch.rand(B,L,D)

>

tensor([[[0.0612, 0.7385],
         [0.7675, 0.3444],
         [0.9129, 0.7601],
         [0.0567, 0.5602]],

        [[0.5450, 0.3749],
         [0.4212, 0.9243],
         [0.1965, 0.9654],
         [0.7230, 0.6295]]])

idx = torch.randint(0, L, size = (B,))

>

tensor([3, 0])

N = get_N(M, idx)

Expected output:

>

tensor([[0.0567, 0.5602],
       [0.5450, 0.3749]])

Thanks.

Output:
import torch

B,L,D = 2,4,2

def get_N(M, idx):
    return M[torch.arange(B), idx, :].squeeze()

M = torch.tensor([[[0.0612, 0.7385],
                   [0.7675, 0.3444],
                   [0.9129, 0.7601],
                   [0.0567, 0.5602]],

                   [[0.5450, 0.3749],
                   [0.4212, 0.9243],
                   [0.1965, 0.9654],
                   [0.7230, 0.6295]]])
idx = torch.tensor([3,0])
N = get_N(M, idx)
print(N)

result:

tensor([[0.0567, 0.5602],
        [0.5450, 0.3749]])

slice along two dimensions.


Instruction: How to split and load huge dataset that doesn't fit into memory into pytorch Dataloader?

Input:
I'm training a deep learning model to do multi label classification of diseases in NIH's Chest Xray-14 dataset using Google's Colab. I can't load all images into Dataloader at once, given around 112k training examples and limited RAM.

Is there a way to just store path of images in pytorch's DataLoader, reading only those images needed for current iteration during training, and once iteration is complete, the images are unloaded from memory, so on so forth until one epoch is complete.

Output:
Yes, the default behavior for the ImageFolder is to create a list of image paths and load the actual images only when needed. It doesn't support multiclass labels. However, you can write your own Dataset to support multi-label, referencing the ImageFolder class for details.

During __init__ you construct a list of image paths and a corresponding list of labels. Images should only be loaded only when __getitem__ is invoked. Below is a stub of such a dataset class, the details will depend on the organization of your files, image types, and label format.

class CustomDataset(torch.utils.data.Dataset):
    def __init__(self, args):
        """ Construct an indexed list of image paths and labels """

    def __getitem__(self, n):
        """ Load image n in the list of image paths and return it along with its label.
            In the case of multiclass the label will probably be a list of values"""

    def __len__(self):
        """ return the total number of images in this dataset """

Once you've created a valid dataset instance an instance of DataLoader should be created, providing your dataset as an argument. A DataLoader is responsible for sampling its dataset, i.e. invoking the __getitem__ method you wrote, and putting individual samples into mini-batches. It also handles parallelized loading and defines how the indices are sampled. The DataLoader itself doesn't store more than it needs. The maximum number of samples it should hold in memory at any time is batch_size * num_workers (or batch_size if num_workers == 0).


Instruction: For each layer in neural networks (pytorch), how many biases should be there?

Input:
I have a simple model in pytorch.

model = Network()

It's details are:

Network(
  (hidden): Linear(in_features=784, out_features=256, bias=True)
  (output): Linear(in_features=256, out_features=10, bias=True)
  (sigmoid): Sigmoid()
  (softmax): Softmax(dim=1)
)

There are 3 neurons' layers in total. 1 input(786 neurons), 1 hidden(256 neurons) and 1 output (10 neurons). Therefore there'll be two weight layers. So there must be two biases (simply two floating point numbers) for both weight layers right? (correct me if i am wrong).

Now after initializing my network i was curious about the two bias values. So i wanted to check the bias value of my hidden layer so i wrote:

model.hidden.bias

And what i got as the result was not i expected! I actually expected one value! And this is what i actually got:

tensor([-1.6868e-02, -3.5661e-02,  1.2489e-02, -2.7880e-02,  1.4025e-02,
        -2.6085e-02,  1.2625e-02, -3.1748e-02,  5.0335e-03,  3.8031e-03,
        -3.1648e-02, -3.4881e-02, -2.0026e-02,  1.9728e-02,  6.2461e-03,
         9.3936e-04, -5.9270e-03, -2.7183e-02, -1.9850e-02, -3.5693e-02,
        -1.9393e-02,  2.6555e-02,  2.3482e-02,  2.1230e-02, -2.2175e-02,
        -2.4386e-02,  3.4848e-02, -2.6044e-02,  1.3575e-02,  9.4125e-03,
         3.0012e-02, -2.6078e-02,  7.1615e-05, -1.7061e-02,  6.6355e-03,
        -3.4966e-02,  2.9311e-02,  1.4060e-02, -2.5763e-02, -1.4020e-02,
         2.9852e-02, -7.9176e-03, -1.8396e-02,  1.6927e-02, -1.1001e-03,
         1.5595e-02,  1.2169e-02, -1.2275e-02, -2.9270e-03, -6.5685e-04,
        -2.4297e-02,  3.0048e-02,  2.9692e-03, -2.5398e-02,  2.9955e-03,
        -9.3653e-04, -1.2932e-02,  2.4232e-02, -3.5182e-02, -1.6163e-02,
         3.0025e-02,  3.1227e-02, -8.2498e-04,  2.7102e-02, -2.3830e-02,
        -3.4958e-02, -1.1886e-02,  1.6097e-02,  1.4579e-02, -2.6744e-02,
         1.1900e-02, -3.4855e-02, -4.2208e-03, -5.2035e-03,  1.7055e-02,
        -4.8580e-03,  3.4088e-03,  1.6923e-02,  3.5570e-04, -3.0478e-02,
         8.4647e-03,  2.5704e-02, -2.3255e-02,  6.9396e-03, -1.2521e-03,
        -9.4101e-03, -2.5798e-02, -1.4438e-03, -7.2684e-03,  3.5417e-02,
        -3.4388e-02,  1.3706e-02, -5.1430e-03,  1.6174e-02,  1.8135e-03,
        -2.9018e-02, -2.9083e-02,  7.4100e-03, -2.7758e-02,  2.4367e-02,
        -3.8350e-03,  9.4390e-03, -1.0844e-02,  1.6381e-02, -2.5268e-02,
         1.3553e-02, -1.0545e-02, -1.3782e-02,  2.8519e-02,  2.3630e-02,
        -1.9703e-02, -2.0147e-02, -1.0485e-02,  2.4637e-02,  1.9989e-02,
         5.6601e-03,  1.9121e-02, -1.5286e-02,  2.5996e-02, -2.9833e-02,
        -2.9458e-02,  2.3944e-02, -3.0107e-02, -1.2307e-02, -1.8419e-02,
         3.3551e-02,  1.2396e-02,  2.9356e-02,  3.3274e-02,  5.4677e-03,
         3.1715e-02,  1.3361e-02,  3.3042e-02,  2.7843e-03,  2.2837e-02,
        -3.4981e-02,  3.2355e-02, -2.7658e-03,  2.2184e-02, -2.0203e-02,
        -3.3264e-02, -3.4858e-02,  1.0820e-03, -1.4279e-02, -2.8041e-02,
         4.1962e-03,  2.4266e-02, -3.5704e-02, -2.6172e-02,  2.3335e-02,
         2.0657e-02, -3.0387e-03, -5.7096e-03, -1.1062e-02,  1.3450e-02,
        -3.3965e-02,  1.9623e-03, -2.0067e-02, -3.3858e-02, -2.1931e-02,
        -1.5414e-02,  2.4454e-02,  2.5668e-02, -1.1932e-02,  5.7540e-04,
         1.5130e-02,  1.3916e-02, -2.1521e-02, -3.0575e-02,  1.8841e-02,
        -2.3240e-02, -2.7297e-02, -3.2668e-02, -1.5544e-02, -5.9408e-03,
         3.0241e-02,  2.2039e-02, -2.4389e-02,  3.1703e-02,  3.5305e-02,
        -2.7501e-03,  2.0154e-02, -5.3489e-03,  1.4177e-02,  1.6829e-02,
         3.3066e-02, -1.3425e-02, -3.2565e-02,  6.5624e-03, -1.5681e-02,
         2.3047e-02,  6.5880e-03, -3.3803e-02,  2.3790e-02, -5.5061e-03,
         2.9413e-02,  1.2290e-02, -1.0958e-02,  1.2680e-03,  1.3343e-02,
         6.6689e-03, -2.2975e-03, -1.2068e-02,  1.6523e-02, -3.1612e-02,
        -1.7529e-02, -2.2220e-02, -1.4723e-02, -1.3495e-02, -5.1805e-03,
        -2.9620e-02,  3.0571e-02, -3.0999e-02,  3.3681e-03,  1.3579e-02,
         1.4837e-02,  1.5694e-02, -1.1178e-02,  4.6233e-03, -2.2583e-02,
        -3.5281e-03,  3.0918e-02,  2.6407e-02,  1.5822e-04, -3.0181e-03,
         8.6989e-03,  2.8998e-02, -1.5975e-02, -3.1574e-02, -1.5609e-02,
         1.0472e-02,  5.8976e-03,  7.0131e-03, -3.2047e-02,  2.6045e-02,
        -2.8882e-02, -2.2121e-02, -3.2960e-02,  1.8268e-02,  3.0984e-02,
         1.4824e-02,  3.0010e-02, -5.7523e-03, -2.0017e-02,  4.8700e-03,
         1.4997e-02, -1.4898e-02,  6.8572e-03,  9.7713e-03,  1.3410e-02,
         4.9619e-03,  3.1016e-02,  3.1240e-02, -3.0203e-02,  2.1435e-02,
         2.7331e-02], requires_grad=True)

Can someone explain to me this behaviour? Why did i get 256 values instead of one?

Edit1:

Here is my understanding of the layers:
For a whole layer of neurons, a bias is just a single value. Am i right? But what i am seeing as the output about are 256 values ? why ? did pytorch assume that i have a bias with each neuron ? is that okay?

Output:
So first it's important to realize what's going on inside of one of these layers. When you write:

Linear(in_features=784, out_features=256, bias=True)

You are modeling a linear relationship between the input and the output. You're probably familiar with this from basic math:

Y = MX + B

However instead of a "slope" and a "y-intercept", you have a weights matrix and a bias term. This is still a linear relationship, but with matrices as our input and output.

Y is our output, M is our weights matrix, X is our input, and B is our Bias. You define that the input is a (N x 784) matrix, and our output is a (N x 256) matrix (N is the number of samples).

If you're familiar with matrix multiplication this means that our weights matrix is (784 X 256). The output of MX will be a (N x 256) matrix, so our bias term must also be (N x 256) for the MX + B to work out.

In general the number of values in the bias term will be the same as the number of out_features.


Instruction: Understanding the use of classes when training a neural network with Pytorch

Input:
I'm currently reading about using the cross-entropy loss to train a neural network in the pytorch documentation. The criterion that is to be used to calculate the loss is called as follows

criterion = nn.CrossEntropyLoss()

According to the torch.nn documentation, CrossEntropyLoss is a class. From my understanding this would mean that criterion is an object of type nn.CrossEntropyLoss.

When training the neural network, criterion is used to calculate the loss in the following way

loss = criterion(input, target)

This is somewhat confusing to me. If criterion is an object, then how can it be used as a function? What I would expect is something like

loss = criterion.calculate_loss(input, target)

where calculate_loss() would be a method defined in the nn.CrossEntropyLoss class. Furthermore, the documentation also uses the following line of code

running_loss += loss.item()

Where does this item() method come from? I can't find a source mentioning item() online.

Output:
  If criterion is an object, then how can it be used as a function?

The criterion object has a forward method that is called in that case. criterion(input, target) is shorthand for criterion.forward(input, target)

  Where does this item() method come from?

This method returns a one dimensional Tensor. The single value can be accessed as a number with item().


Instruction: Why does torch.gt function turn requires_grad into False?

Input:
The requires_grad of tensor b and c are True. But the requires_grad of tensor d is False. I am curious why this change happens because all the requires_grad of inputs are True.

However, the requires_grad of tensor e is True. I can still do backward() on e. But is there an error in this way?

I am using Python3.7 and Pytorch1.1.

import torch
import torch.nn as nn

net = nn.Conv2d(1, 1, 3, padding=1)
a = torch.randn(1, 1, 10, 10)
b = net(a)
c = net(b)

d = torch.gt(b, c)

e = b - c
e[e > 0] = 1.0
e[e < 0] = 0.0

Output:
I assume this is because you cannot take a gradient of greater than operation. The return type is boolean:

>>> torch.gt(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]]))
tensor([[False, True], [False, False]])

Whereas minus or other arithmetic operation returns another numeral.


Instruction: Torchvision normalize - how it operates on tuple of means/sds?

Input:
I don't understand how this transform works from torchvision. Ultimately I want to build a custom normalize class so I need to figure out how this works first.

Here in the docs it describes the init like this:

def __init__(self, mean, std, inplace=False):
        self.mean = mean
        self.std = std
        self.inplace = inplace

And when I pass these parameters usually (not custom class) I pass them as a list or tuple for each channel:

transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])

But if I look at the call:

return F.normalize(tensor, self.mean, self.std, self.inplace)

All this passes the tuple to is F.normalize() which only accepts a single value for the p parameter.

The class must iterate through the channels somehow to allow this to be implemented but how does it do this and how can I implement it in custom class?

Based on this tutorial, I would describe it like this:

class Normalize(object):
    """Convert ndarrays in sample to Tensors."""

    def __init__(self, mean, std, inplace=False):
        self.mean = mean
        self.std = std
        self.inplace = inplace

    def __call__(self, sample):
        image, landmarks = sample['image'], sample['landmarks']
        return {'image': F.normalize(image, self.mean, self.std, self.inplace),
                'landmarks': landmarks}

But this does not work because it does not go through each channel.

Output:
The normalize function called in there is this one https://github.com/pytorch/vision/blob/master/torchvision/transforms/functional.py#L191

The input is a tensor of shape (C, H, W) and mean and std can be sequences, that are internally converted to tensors. The normalization is done through broadcasting in this way:

tensor.sub_(mean[:, None, None]).div_(std[:, None, None])


Instruction: Would it possible to use all memory of GPUs with one model?

Input:
There is a model and two GPUs. I put the model on GPU with model.cuda(). If I passed a big image to the model, it allocated all memory of GPU0 and then it raised CUDA out of memory error without allocating any memory of GPU1.

Because there is only one image every forward(), I can not use such torch.nn.DataParallel things to split input.

Is there any way to use all the memory of GPUs when passing a image to the model?

I'm using Python3.7 and Pytorch1.1.

Output:
you can split your model into two submodule.
like this:

class MyModel(nn.Module):
    def __init__(self, split_gpus):
        self.large_submodule1 = ...
        self.large_submodule2 = ...

        self.split_gpus = split_gpus
        if split_gpus:
            self.large_submodule1.cuda(0)
            self.large_submodule1.cuda(1)

    def forward(self, x):
        x = self.large_submodule1(x)
        if split_gpus:
            x = x.cuda(1) # P2P GPU transfer
        return self.large_submodule2(x)

from the pytorch discuss


Instruction: Optimizing docker image size

Input:
I am building a docker image for my python app using the base image python:3.5-slim-buster.

I am running the below command inside the Dockerfile:

RUN pip install --no-cache-dir -r requirements.txt which has torch library in the requirements file.

After building the image it is having a size of 2.29 GB. But if I build the image without torch inside the requirements file, it has only ~900 MB.

when I manually run the image and check inside the container:

The torch (/usr/local/lib/python3.5/site-packages/torch) directory is 1.3GB.

So even if I do a multistage build and try to copy the contents from
/usr/local/lib/python3.5/site-packages to a new image, I suppose it is not going to help me.

Is there any other standard optimization practice which can help me to reduce the image size?

Output:
Assuming you want a trained model of some kind from pytorch (usually neural network) you should indeed use multistage Docker build as follows (IMO at least):

1. Training model and exporting

Write your script as per usual, including all dependencies you need. Train your model and save it as artifact using torchscript's torch.jit.script.

2. Second stage

Using C++ write inference code loading your net and compile the source (use libtorch).

3. Final image

Copy binary from previous step and put it is as Docker's entrypoint so you can run it as disposable neural net within container.


Instruction: Numpy random.choice probablities don't sum up to 1

Input:
I wanna randomly select sample points based on the probability distribution specified by prob for a given row. However, I get the error in np.random.choice that the probabilities don't add up to 1. This is very weird because I first normalize using the L1-norm along the rows and then I define a uniform distribution if the values are smaller than the threshold 1e-6.

import numpy as np
import torch.nn.functional as F

prob = F.normalize(outputs, p=1, dim=1).clone().data.cpu().numpy() # outputs is a torch.Tensor of shape (14, 6890)
all_zero = np.where(prob.max(1) < 1e-6)[0] # find indices of rows where all values are smaller
prob[all_zero] = np.full(prob.shape[1], 1 / prob.shape[1]) # fill those rows uniformly
# ... somewhere later inside a method
for j in range(14):
    sample = np.random.choice(6890, 4, replace=False, p=prob[j])

Do you understand, why that is?

Output:
As error suggests prob[j] doesn't sum to 1.

Your epsilon 1e-6 is way too big to be considered insignificant, there is no need for this operation at all. If you insist, you have to redistribute zero-ed out values across what's left to 1 (and it seems you did just that actually).

All in all you didn't normalize the array to 1:

prob /= prob.sum(axis=1) # make it prob dist

BTW. Broadcasting will extend your single number to the whole row, no need for np.full:

prob[all_zero] = 1 / prob.shape[1]


Instruction: Caffe2 doesn't seem to see cuDNN?

Input:
I am trying to install SpConv (spatially sparse convolution library), however when running python3 setup.py bdist_wheel, I get an error which seems to be related to Caffe2 not being able to see cuDNN, as I infer from this message:

CMake Error at /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Caffe2/Caffe2Config.cmake:96 (message):
  Your installed Caffe2 version uses cuDNN but I cannot find the cuDNN
  libraries.  Please set the proper cuDNN prefixes and / or install cuDNN.

I have tried reinstalling cuDNN but no luck. Can I manually point Caffe2 to cuDNN? This is because usually cuda is located in /usr/local/cuda/, however I have it in /usr/lib/cuda.

Full output below:

running bdist_wheel
running build
running build_py
running build_ext
Release
|||||CMAKE ARGS||||| ['-DCMAKE_PREFIX_PATH=/home/ivan/.local/lib/python3.6/site-packages/torch', '-DPYBIND11_PYTHON_VERSION=3.6', '-DSPCONV_BuildTests=OFF', '-DCMAKE_CUDA_FLAGS="--expt-relaxed-constexpr" -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=/mnt/home/ivan/second.pytorch/second/spconv/build/lib.linux-x86_64-3.6/spconv', '-DCMAKE_BUILD_TYPE=Release']
-- Caffe2: CUDA detected: 9.1
-- Caffe2: CUDA nvcc is: /usr/bin/nvcc
-- Caffe2: CUDA toolkit directory: /usr
-- Caffe2: Header version is: 9.1
-- Could NOT find CUDNN (missing: CUDNN_INCLUDE_DIR)
CMake Warning at /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Caffe2/public/cuda.cmake:131 (message):
  Caffe2: Cannot find cuDNN library.  Turning the option off
Call Stack (most recent call first):
  /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Caffe2/Caffe2Config.cmake:88 (include)
  /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Torch/TorchConfig.cmake:40 (find_package)
  CMakeLists.txt:20 (find_package)

-- Autodetected CUDA architecture(s):  6.1
-- Added CUDA NVCC flags for: -gencode;arch=compute_61,code=sm_61
CMake Error at /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Caffe2/Caffe2Config.cmake:96 (message):
  Your installed Caffe2 version uses cuDNN but I cannot find the cuDNN
  libraries.  Please set the proper cuDNN prefixes and / or install cuDNN.
Call Stack (most recent call first):
  /home/ivan/.local/lib/python3.6/site-packages/torch/share/cmake/Torch/TorchConfig.cmake:40 (find_package)
  CMakeLists.txt:20 (find_package)

-- Configuring incomplete, errors occurred!
See also "/mnt/home/ivan/second.pytorch/second/spconv/build/temp.linux-x86_64-3.6/CMakeFiles/CMakeOutput.log".
See also "/mnt/home/ivan/second.pytorch/second/spconv/build/temp.linux-x86_64-3.6/CMakeFiles/CMakeError.log".
Traceback (most recent call last):
  File "setup.py", line 99, in <module>
    zip_safe=False,
  File "/home/ivan/.local/lib/python3.6/site-packages/setuptools/__init__.py", line 145, in setup
    return distutils.core.setup(**attrs)
  File "/usr/lib/python3.6/distutils/core.py", line 148, in setup
    dist.run_commands()
  File "/usr/lib/python3.6/distutils/dist.py", line 955, in run_commands
    self.run_command(cmd)
  File "/usr/lib/python3.6/distutils/dist.py", line 974, in run_command
    cmd_obj.run()
  File "/usr/lib/python3/dist-packages/wheel/bdist_wheel.py", line 204, in run
    self.run_command('build')
  File "/usr/lib/python3.6/distutils/cmd.py", line 313, in run_command
    self.distribution.run_command(command)
  File "/usr/lib/python3.6/distutils/dist.py", line 974, in run_command
    cmd_obj.run()
  File "/usr/lib/python3.6/distutils/command/build.py", line 135, in run
    self.run_command(cmd_name)
  File "/usr/lib/python3.6/distutils/cmd.py", line 313, in run_command
    self.distribution.run_command(command)
  File "/usr/lib/python3.6/distutils/dist.py", line 974, in run_command
    cmd_obj.run()
  File "setup.py", line 40, in run
    self.build_extension(ext)
  File "setup.py", line 82, in build_extension
    subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
  File "/usr/lib/python3.6/subprocess.py", line 311, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['cmake', '/mnt/home/ivan/second.pytorch/second/spconv', '-DCMAKE_PREFIX_PATH=/home/ivan/.local/lib/python3.6/site-packages/torch', '-DPYBIND11_PYTHON_VERSION=3.6', '-DSPCONV_BuildTests=OFF', '-DCMAKE_CUDA_FLAGS="--expt-relaxed-constexpr" -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=/mnt/home/ivan/second.pytorch/second/spconv/build/lib.linux-x86_64-3.6/spconv', '-DCMAKE_BUILD_TYPE=Release']' returned non-zero exit status 1.

System: Ubuntu 18.04, CUDA 9.1, cuDNN 7.6.3

Output:
Found the problem. I needed to set CUDNN_INCLUDE_DIR to /usr/lib/cuda/include (i.e. where the cudnn.h file is located).

Lesson learnt: take time to understand the error, missing: CUDNN_INCLUDE_DIR was key.


Instruction: pytorch questions: how to add bias term and extract its value? class vs sequential model? and softmax

Input:
I have a basic neural network model in pytorch like this:

import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.sigmoid = nn.Sigmoid()
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        out = self.fc1(x)
        out = self.sigmoid(out)
        out = self.fc2(out)
        return out
net = Net(400, 512,10)

How can I extract bias/intercept term from net.parameters()?
And is this model equivalent to using sequential()?

net = nn.Sequential(nn.Linear(input_dim, hidden_dim[0]),
                      nn.Sigmoid(),
                      nn.Linear(hidden_dim[0], hidden_dim[1]),
                      nn.Sigmoid(),
                      nn.Linear(hidden_dim[1], output_dim))

Is nn.Softmax() optional at the end of either model for multi-class classification? If I understand correctly, with software it outputs probability of a certain class but without it returns predicted output?

Thanks in advance for answering my newbie questions.

Output:
Let's answer questions one by one. is this model equivalent to using sequential()
Short answer: No. You can see that you have added two Sigmoid and two linear layers. You can print your net and see the result:

net = Net(400, 512,10)

print(net.parameters())
print(net)
input_dim = 400
hidden_dim = 512
output_dim = 10

model = Net(400, 512,10)

net = nn.Sequential(nn.Linear(input_dim, hidden_dim),
                      nn.Sigmoid(),
                      nn.Linear(hidden_dim, hidden_dim),
                      nn.Sigmoid(),
                      nn.Linear(hidden_dim, output_dim))

print(net)

The output is:

Net(
  (fc1): Linear(in_features=400, out_features=512, bias=True)
  (sigmoid): Sigmoid()
  (fc2): Linear(in_features=512, out_features=10, bias=True)
)

Sequential(
  (0): Linear(in_features=400, out_features=512, bias=True)
  (1): Sigmoid()
  (2): Linear(in_features=512, out_features=512, bias=True)
  (3): Sigmoid()
  (4): Linear(in_features=512, out_features=10, bias=True)
)

I hope you can see where they differ.

Your first question: How can I extract bias/intercept term from net.parameters()

The answer:

model = Net(400, 512,10)

bias = model.fc1.bias

print(bias)

the output is:

tensor([ 3.4078e-02,  3.1537e-02,  3.0819e-02,  2.6163e-03,  2.1002e-03,
         4.6842e-05, -1.6454e-02, -2.9456e-02,  2.0646e-02, -3.7626e-02,
         3.5531e-02,  4.7748e-02, -4.6566e-02, -1.3317e-02, -4.6593e-02,
        -8.9996e-03, -2.6568e-02, -2.8191e-02, -1.9806e-02,  4.9720e-02,
        ---------------------------------------------------------------
        -4.6214e-02, -3.2799e-02, -3.3605e-02, -4.9720e-02, -1.0293e-02,
         3.2559e-03, -6.6590e-03, -1.2456e-02, -4.4547e-02,  4.2101e-02,
        -2.4981e-02, -3.6840e-03], requires_grad=True)


Instruction: Training with BatchNorm in pytorch

Input:
I'm wondering if I need to do anything special when training with BatchNorm in pytorch. From my understanding the gamma and beta parameters are updated with gradients as would normally be done by an optimizer. However, the mean and variance of the batches are updated slowly using momentum.

So do we need to specify to the optimizer when the mean and variance parameters are updated, or does pytorch automatically take care of this?
Is there a way to access the mean and variance of the BN layer so that I can make sure it was changing while I trained the model.

If needed here is my model and training procedure:

def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0.):
    "Sequence of batchnorm (if `bn`), dropout (with `p`) and linear (`n_in`,`n_out`) layers followed by `actn`."
    layers = [nn.BatchNorm1d(n_in)] if bn else []
    if p != 0: layers.append(nn.Dropout(p))
    layers.append(nn.Linear(n_in, n_out))

    return nn.Sequential(*layers)

class Model(nn.Module):
    def __init__(self, i, o, h=()):
        super().__init__()

        nodes = (i,) + h + (o,)
        self.layers = nn.ModuleList([bn_drop_lin(i,o, p=0.5)
                                     for i, o in zip(nodes[:-1], nodes[1:])])

    def forward(self, x):
        x = x.view(x.shape[0], -1)
        for layer in self.layers[:-1]:
            x = F.relu(layer(x))

        return self.layers[-1](x)

Training:

for i, data in enumerate(trainloader):
    # get the inputs; data is a list of [inputs, labels]
    inputs, labels = data

    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = net(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

Output:
Batchnorm layers behave differently depending on if the model is in train or eval mode.
When net is in train mode (i.e. after calling net.train()) the batch norm layers contained in net will use batch statistics along with gamma and beta parameters to scale and translate each mini-batch. The running mean and variance will also be adjusted while in train mode. These updates to running mean and variance occur during the forward pass (when net(inputs) is called). The gamma and beta parameters are like any other pytorch parameter and are updated only once optimizer.step() is called.
When net is in eval mode (net.eval()) batch norm uses the historical running mean and running variance computed during training to scale and translate samples.
You can check the batch norm layers running mean and variance by displaying the layers running_mean and running_var members to ensure batch norm is updating them as expected. The learnable gamma and beta parameters can be accessed by displaying the weight and bias members of a batch norm layer respectively.
Edit
Below is a simple demonstration code showing that running_mean is updated during forward. Observe that it is not updated by the optimizer.
>>> import torch
>>> import torch.nn as nn
>>> layer = nn.BatchNorm1d(5)
>>> layer.train()
>>> layer.running_mean
tensor([0., 0., 0., 0., 0.])
>>> result = layer(torch.randn(5,5))
>>> layer.running_mean
tensor([ 0.0271,  0.0152, -0.0403, -0.0703, -0.0056])


Instruction: Simple Conv Net

Input:
I'm new on pytorch and I would like your light on a small net I try to setup.

class PzConv2d(nn.Module):
    """ Convolution 2D Layer followed by PReLU activation
    """
    def __init__(self, n_in_channels, n_out_channels, **kwargs):
        super(PzConv2d, self).__init__()
        self.conv = nn.Conv2d(n_in_channels, n_out_channels, bias=True,
                            **kwargs)
        self.activ = nn.ReLU()

    def forward(self, x):
        x = self.conv(x)
        return self.activ(x)

class PzPool2d(nn.Module):
    """ Average Pooling Layer
    """
    def __init__(self, kernel_size, stride, padding=0):
        super(PzPool2d, self).__init__()
        self.pool = nn.AvgPool2d(kernel_size=kernel_size,
                                 stride=stride,
                                 padding=padding,
                                 ceil_mode=True,
                                 count_include_pad=False)

    def forward(self, x):
        return self.pool(x)

class PzFullyConnected(nn.Module):
    """ Dense or Fully Connected Layer followed by ReLU
    """
    def __init__(self, n_inputs, n_outputs, withrelu=True, **kwargs):
        super(PzFullyConnected, self).__init__()
        self.withrelu = withrelu
        self.linear = nn.Linear(n_inputs, n_outputs, bias=True)
        self.activ = nn.ReLU()

    def forward(self, x):
        x = self.linear(x)
        if self.withrelu:
            x = self.activ(x)
        return x

class NetCNN(nn.Module):
    def __init__(self,n_input_channels,debug=False):
        super(NetCNN, self).__init__()

        self.n_bins = 180
        self.debug = debug
        self.conv0 = PzConv2d(n_in_channels=n_input_channels,
                              n_out_channels=64,
                              kernel_size=5,padding=2)
        self.pool0 = PzPool2d(kernel_size=2,stride=2,padding=0)

        self.conv1 = PzConv2d(n_in_channels=64,
                              n_out_channels=92,
                              kernel_size=3,padding=2)
        self.pool1 = PzPool2d(kernel_size=2,stride=2,padding=0)

        self.conv2 = PzConv2d(n_in_channels=92,
                              n_out_channels=128,
                              kernel_size=3,padding=2)
        self.pool2 = PzPool2d(kernel_size=2,stride=2,padding=0)

        self.fc0 = PzFullyConnected(n_inputs=12800,n_outputs=1024)
        self.fc1 = PzFullyConnected(n_inputs=1024,n_outputs=self.n_bins)

    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

    def forward(self, x, dummy):
        # x:image tensor N_batch, Channels, Height, Width
        #    size N, Channels:=5 filtres, H,W = 64 pixels
        # dummy: is not used

        # stage 0 conv 64 x 5x5
        x = self.conv0(x)
        x = self.pool0(x)

        # stage 1 conv 92 x 3x3
        x = self.conv1(x)
        x = self.pool1(x)

        # stage 2 conv 128 x 3x3
        x = self.conv2(x)
        x = self.pool2(x)

        x = self.fc0(x.view(-1,self.num_flat_features(x)))
        x = self.fc1(x)

        output = x

        return output

I have checked that the dimensions of the intermediate "x" tensor in the forward process is well shaped (at least when I send a random input image tensor). But if you see something strange let me know, please.

Now, I have seen code with F."function" sequence in the forward method, instead of declaring the different layers as I have done. Does it make a difference?

(Notice that I'm using  F.cross_entropy as loss function, so I do not end may network by a SoftMax.)

Thanks.

Output:
As you can see from pytorch doc there are "layers" and there are "functionals". You already noticed that they are quite similar, but there is a difference:
A layer is often more than just a "functional" it also wraps around trainable parameters.
Therefore, you can have a F.conv2d(...) in your forward() function, but you will have to manually provide (store/update) the weights/kernel for this convolution. On the other hand, if you are using nn.Conv2d pytorch takes care of managing/storing/updating the weights/kernels for you.
Some layers have no internal parameters/buffers (e.g., nn.ReLU, nn.Softmax etc.) thus you can choose if you want to have a "layer" for this operation or only calling the appropriate functional in your forward() function. It is a matter of convenience and habits, it's up to you.


Instruction: Memory error when iterate over two dataloaders simultaneously in pytorch

Input:
I am trying to train my model using 2 dataloaders from 2 different datasets.

I found how to set up this by using cycle() and zip() because my datasets are not the same length from here: How to iterate over two dataloaders simultaneously using pytorch?

  File "/home/Desktop/example/train.py", line 229, in train_2
    for i, (x1, x2) in enumerate(zip(cycle(train_loader_1), train_loader_2)):
  File "/home/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 346, in __next__
    data = self.dataset_fetcher.fetch(index)  # may raise StopIteration
  File "/home/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/home/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 80, in default_collate
    return [default_collate(samples) for samples in transposed]
  File "/home/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 80, in <listcomp>
    return [default_collate(samples) for samples in transposed]
  File "/home/.conda/envs/3dcnn/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 56, in default_collate
    return torch.stack(batch, 0, out=out)
RuntimeError: [enforce fail at CPUAllocator.cpp:64] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 154140672 bytes. Error code 12 (Cannot allocate memory)

I tried to solve that by setting num_workers=0, decreasing the batch size, using pinned_memory=False and shuffle=False...
But none of it worked... I am having 256GB of RAM and 4 NVIDIA TESLA V100 GPUs.

I tried to run it just by not training in 2 dataloaders simultaneously but individually and it worked. However for my project I need this parallel training with 2 datasets...

Output:
Based on this discussion, instead of cycle() and zip() I avoid any errors by using:

  try:
     data, target = next(dataloader_iterator)
  except StopIteration:
     dataloader_iterator = iter(dataloader)
     data, target = next(dataloader_iterator)

kudos to @srossi93 from this pytorch post!


Instruction: PyTorch: Dataloader for time series task

Input:
I have a Pandas dataframe with n rows and k columns loaded into memory. I would like to get batches for a forecasting task where the first training example of a batch should have shape (q, k) with q referring to the number of rows from the original dataframe (e.g. 0:128). The next example should be (128:256, k) and so on. So, ultimately, one batch should have the shape (32, q, k) with 32 corresponding to the batch size.
Since TensorDataset from data_utils does not work here, I am wondering what the best way would be. I tried to use np.array_split() to get as first dimension the number of possible splits of q values in order to write a custom DataLoader but then reshaping is not guaranteed to work since not all arrays have the same shape.
Here is a minimal example to make it more clear. In this case, batch size is 3 and q is 2:
import pandas as pd
import numpy as np
df = pd.DataFrame(data=np.arange(0,30).reshape(10,3),columns=['A','B','C'])

The dataset:
    A   B   C
0   0   1   2
1   3   4   5
2   6   7   8
3   9   10  11
4   12  13  14
5   15  16  17
6   18  19  20
7   21  22  23
8   24  25  26
9   27  28  29

The first batch in this case should have the shape (3,2,3) and look like:
array([[[ 0.,  1.,  2.],
        [ 3.,  4.,  5.]],

       [[ 3.,  4.,  5.],
        [ 6.,  7.,  8.]],

       [[ 6.,  7.,  8.],
        [ 9., 10., 11.]]])

Output:
You can write your analog of the TensorDataset. To do this you need to inherit from the Dataset class.

from torch.utils.data import Dataset, DataLoader

class MyDataset(Dataset):
    def __init__(self, data_frame, q):
        self.data = data_frame.values
        self.q = q

    def __len__(self):
        return self.data.shape[0] // self.q

    def __getitem__(self, index):
        return self.data[index * self.q: (index+1) * self.q]


Instruction: How to repeat tensor in a specific new dimension in PyTorch

Input:
If I have a tensor A which has shape [M, N],
I want to repeat the tensor K times so that the result B has shape [M, K, N]
and each slice B[:, k, :] should has the same data as A.
Which is the best practice without a for loop.
K might be in other dimension.

torch.repeat_interleave() and tensor.repeat() does not seem to work. Or I am using it in a wrong way.

Output:
tensor.repeat should suit your needs but you need to insert a unitary dimension first. For this we could use either tensor.unsqueeze  or tensor.reshape. Since unsqueeze is specifically defined to insert a unitary dimension we will use that.
B = A.unsqueeze(1).repeat(1, K, 1)

Code Description A.unsqueeze(1) turns A from an [M, N] to [M, 1, N] and .repeat(1, K, 1) repeats the tensor K times along the second dimension.


Instruction: Is it possible to load a pretrained Pytorch model from a GCS bucket URL without first persisting locally?

Input:
I'm asking this in the context of Google Dataflow, but also generally.

Using PyTorch, I can reference a local directory containing multiple files that comprise a pretrained model. I happen to be working with a Roberta model, but the interface is the same for others.

ls some-directory/
      added_tokens.json
      config.json
      merges.txt
      pytorch_model.bin
      special_tokens_map.json vocab.json

from pytorch_transformers import RobertaModel

# this works
model = RobertaModel.from_pretrained('/path/to/some-directory/')

However, my pretrained model is stored in a GCS bucket. Let's call it gs://my-bucket/roberta/.

In the context of loading this model in Google Dataflow, I'm trying to remain stateless and avoid persisting to disk, so my preference would be to get this model straight from GCS. As I understand it, the PyTorch general interface method from_pretrained() can take the string representation of a local dir OR a URL. However, I can't seem to load the model from a GCS URL.

# this fails
model = RobertaModel.from_pretrained('gs://my-bucket/roberta/')
# ValueError: unable to parse gs://mahmed_bucket/roberta-base as a URL or as a local path

If I try to use the public https URL of the directory blob, it will also fail, although that is likely due to lack of authentication since the credentials referenced in the python environment that can create clients don't translate to public requests to https://storage.googleapis

# this fails, probably due to auth
bucket = gcs_client.get_bucket('my-bucket')
directory_blob = bucket.blob(prefix='roberta')
model = RobertaModel.from_pretrained(directory_blob.public_url)
# ValueError: No JSON object could be decoded

# and for good measure, it also fails if I append a trailing /
model = RobertaModel.from_pretrained(directory_blob.public_url + '/')
# ValueError: No JSON object could be decoded

I understand that GCS doesn't actually have subdirectories and it's actually just being a flat namespace under the bucket name. However, it seems like I'm blocked by the necessity of authentication and a PyTorch not speaking gs://.

I can get around this by persisting the files locally first.

from pytorch_transformers import RobertaModel
from google.cloud import storage
import tempfile

local_dir = tempfile.mkdtemp()
gcs = storage.Client()
bucket = gcs.get_bucket(bucket_name)
blobs = bucket.list_blobs(prefix=blob_prefix)
for blob in blobs:
    blob.download_to_filename(local_dir + '/' + os.path.basename(blob.name))
model = RobertaModel.from_pretrained(local_dir)

But this seems like such a hack, and I keep thinking I must be missing something. Surely there's a way to stay stateless and not have to rely on disk persistence!

So is there a way to load a pretrained model stored in GCS?
Is there a way to authenticate when doing the public URL request in this context?
Even if there is a way to authenticate, will the non-existence of subdirectories still be an issue?

Thanks for the help! I'm also happy to be pointed to any duplicate questions 'cause I sure couldn't find any.

Edits and Clarifications

My Python session is already authenticated to GCS, which is why I'm able to download the blob files locally and then point to that local directory with load_frompretrained()
load_frompretrained() requires a directory reference because it needs all the files listed at the top of the question, not just pytorch-model.bin
To clarify question #2, I was wondering if there's some way of giving the PyTorch method a request URL that had encrypted credentials embedded or something like that. Kind of a longshot, but I wanted to make sure I hadn't missed anything.
To clarify question #3 (in addition to the comment on one answer below), even if there's a way to embed credentials in the URL that I don't know about, I still need to reference a directory rather than a single blob, and I don't know if the GCS subdirectory would be recognized as such because (as the Google docs state) subdirectories in GCS are an illusion and they don't represent a real directory structure. So I think this question is irrelevant or at least blocked by question #2, but it's a thread I chased for a bit so I'm still curious.

Output:
MAJOR EDIT:

You can install wheel files on Dataflow workers, and you can also use worker temp storage to persist binary files locally!

It's true that (currently as of Nov 2019) you can't do this by supplying a --requirements argument. Instead you have to use setup.py like this. Assume any constants IN CAPS are defined elsewhere.

REQUIRED_PACKAGES = [
    'torch==1.3.0',
    'pytorch-transformers==1.2.0',
]

setup(
    name='project_dir',
    version=VERSION,
    packages=find_packages(),
    install_requires=REQUIRED_PACKAGES)

Run script

python setup.py sdist

python project_dir/my_dataflow_job.py \
--runner DataflowRunner \
--project ${GCP_PROJECT} \
--extra_package dist/project_dir-0.1.0.tar.gz \
# SNIP custom args for your job and required Dataflow Temp and Staging buckets #

And within the job, here's downloading and using the model from GCS in the context of a custom Dataflow operator. For convenience we wrapped a few utility methods in a SEPARATE MODULE (important to get around Dataflow dependency uploads) and imported them at the LOCAL SCOPE of the custom operator, not global.

class AddColumn(beam.DoFn):
    PRETRAINED_MODEL = 'gs://my-bucket/blah/roberta-model-files'

    def get_model_tokenizer_wrapper(self):
        import shutil
        import tempfile
        import dataflow_util as util
        try:
            return self.model_tokenizer_wrapper
        except AttributeError:
            tmp_dir = tempfile.mkdtemp() + '/'
            util.download_tree(self.PRETRAINED_MODEL, tmp_dir)
            model, tokenizer = util.create_model_and_tokenizer(tmp_dir)
            model_tokenizer_wrapper = util.PretrainedPyTorchModelWrapper(
                model, tokenizer)
            shutil.rmtree(tmp_dir)
            self.model_tokenizer_wrapper = model_tokenizer_wrapper
            logging.info(
                'Successfully created PretrainedPyTorchModelWrapper')
            return self.model_tokenizer_wrapper

    def process(self, elem):
        model_tokenizer_wrapper = self.get_model_tokenizer_wrapper()

        # And now use that wrapper to process your elem however you need.
        # Note that when you read from BQ your elements are dictionaries
        # of the column names and values for each BQ row.

Utility functions in SEPARATE MODULE within the codebase. In our case in the project root this was in dataflow_util/init.py but you don't have to do it that way.

from contextlib import closing
import logging

import apache_beam as beam
import numpy as np
from pytorch_transformers import RobertaModel, RobertaTokenizer
import torch

class PretrainedPyTorchModelWrapper():
    def __init__(self, model, tokenizer):
        self.model = model
        self.tokenizer = tokenizer

def download_tree(gcs_dir, local_dir):
    gcs = beam.io.gcp.gcsio.GcsIO()
    assert gcs_dir.endswith('/')
    assert local_dir.endswith('/')
    for entry in gcs.list_prefix(gcs_dir):
        download_file(gcs, gcs_dir, local_dir, entry)

def download_file(gcs, gcs_dir, local_dir, entry):
    rel_path = entry[len(gcs_dir):]
    dest_path = local_dir + rel_path
    logging.info('Downloading %s', dest_path)
    with closing(gcs.open(entry)) as f_read:
        with open(dest_path, 'wb') as f_write:
            # Download the file in chunks to avoid requiring large amounts of
            # RAM when downloading large files.
            while True:
                file_data_chunk = f_read.read(
                    beam.io.gcp.gcsio.DEFAULT_READ_BUFFER_SIZE)
                if len(file_data_chunk):
                    f_write.write(file_data_chunk)
                else:
                    break

def create_model_and_tokenizer(local_model_path_str):
    """
    Instantiate transformer model and tokenizer

      :param local_model_path_str: string representation of the local path
             to the directory containing the pretrained model
      :return: model, tokenizer
    """
    model_class, tokenizer_class = (RobertaModel, RobertaTokenizer)

    # Load the pretrained tokenizer and model
    tokenizer = tokenizer_class.from_pretrained(local_model_path_str)
    model = model_class.from_pretrained(local_model_path_str)

    return model, tokenizer

And there you have it folks! More details can be found here: https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/

What I've discovered is that this whole chain of questioning is irrelevant because Dataflow only allows you to install source distribution packages on workers which means you can't actually install PyTorch.

When you supply a requirements.txt file, Dataflow will install with the --no-binary flag which prevents installation of Wheel (.whl) packages and only allows source distributions (.tar.gz). I decided trying to roll my own source distribution for PyTorch on Google Dataflow where it's half C++ and part Cuda and part who knows what was a fool's errand.

Thanks for the input along the way y'all.


Instruction: How to select specific labels in pytorch MNIST dataset

Input:
I'm trying to create dataloaders using only a specific digit from PyTorch Mnist dataset

I already tried to create my own Sampler but it doesn't work and I'm not sure I'm using correctly the mask.

class YourSampler(torch.utils.data.sampler.Sampler):

    def __init__(self, mask):

        self.mask = mask

    def __iter__(self):

        return (self.indices[i] for i in torch.nonzero(self.mask))

    def __len__(self):

        return len(self.mask)

mnist = datasets.MNIST(root=dataroot, train=True, download=True, transform = transform)

mask = [True if mnist[i][1] == 5 else False for i in range(len(mnist))]

mask = torch.tensor(mask)

sampler = YourSampler(mask)

trainloader = torch.utils.data.DataLoader(mnist, batch_size=4, sampler = sampler, shuffle=False, num_workers=2)

So far I had many different types of errors. For this implementation, it's "Stop Iteration".
I feel like this is very easy/stupid but I can't find a simple way to do it.
Thank you for your help!

Output:
Thank you for your help.
After a while I figured out a solution (but might not be the best at all):

class YourSampler(torch.utils.data.sampler.Sampler):
    def __init__(self, mask, data_source):
        self.mask = mask
        self.data_source = data_source

    def __iter__(self):
        return iter([i.item() for i in torch.nonzero(mask)])

    def __len__(self):
        return len(self.data_source)

mnist = datasets.MNIST(root=dataroot, train=True, download=True, transform = transform)
mask = [1 if mnist[i][1] == 5 else 0 for i in range(len(mnist))]
mask = torch.tensor(mask)
sampler = YourSampler(mask, mnist)
trainloader = torch.utils.data.DataLoader(mnist, batch_size=batch_size,sampler = sampler, shuffle=False, num_workers=workers)


Instruction: Pytorch how to stack tensor like for loop

Input:
I want to concat tensor generated in for loop, and get 2dTensor.
standard python, like below.

li = []
for i in range(0, len(items)):
    # calc something
    li.append(calc_result)

In my case, in for loop, generate torch.Size([768]) Tensor, and I want to get torch.Size([len(item),768]) Tensor.
How to do this?

Output:
You can use torch.stack:

torch.stack(li, dim=0)

after the for loop will give you a torch.Tensor of that size.

Note that if you know in advance the size of the final tensor, you can allocate an empty tensor beforehand and fill it in the for loop:

x = torch.empty(size=(len(items), 768))
for i in range(len(items)):
    x[i] = calc_result

This is usually faster than doing the stack.


Instruction: Replicating Semantic Analysis Model in Demo

Input:
Good day, I am a student that is interested in NLP. I have come across the demo on  AllenNLP's homepage, which stated that:

  The model is a simple LSTM using GloVe embeddings that is trained on the binary classification setting of the Stanford Sentiment Treebank. It achieves about 87% accuracy on the test set.

Is there any reference to the sample code or any tutorial that I can follow to replicate this result, so that I can learn more about this subject? I am trying to obtain a Regression Output (Instead of classification).

I hope that someone can point me in the right direction.. Any help is much appreciated. Thank you!

Output:
AllenAI provides all code for examples and lib opensource on Git, including AllenNLP.

I found exactly how the example was run here: https://github.com/allenai/allennlp/blob/master/allennlp/tests/data/dataset_readers/stanford_sentiment_tree_bank_test.py

However, to make it a Regression task, you'll have to tweak directly on Pytorch, which is the underlying technology for AllenNLP.


Instruction: How to share weights between modules in Pytorch?

Input:
What is the correct way of sharing weights between two layers(modules) in Pytorch?
Based on my findings in the Pytorch discussion forum, there are several ways for doing this.
As an example, based on this discussion, I thought simply assigning the transposed weights would do it. That is doing :
 self.decoder[0].weight = self.encoder[0].weight.t()

This however, proved to be wrong and causes an error.
I then tried wrapping the above line in a nn.Parameter():
self.decoder[0].weight = nn.Parameter(self.encoder[0].weight.t())

This eliminates the error, but then again, there is no sharing happening here. by this I just initialized a new tensor with the same values as the encoder[0].weight.t().
I then found this link which provides different ways for sharing weights. however, I'm skeptical if all methods given there are actually correct.
For example, one way is demonstrated like this :
# tied autoencoder using off the shelf nn modules
class TiedAutoEncoderOffTheShelf(nn.Module):
    def __init__(self, inp, out, weight):
        super().__init__()
        self.encoder = nn.Linear(inp, out, bias=False)
        self.decoder = nn.Linear(out, inp, bias=False)

        # tie the weights
        self.encoder.weight.data = weight.clone()
        self.decoder.weight.data = self.encoder.weight.data.transpose(0,1)

    def forward(self, input):
        encoded_feats = self.encoder(input)
        reconstructed_output = self.decoder(encoded_feats)
        return encoded_feats, reconstructed_output

Basically it creates a new weight tensor using nn.Parameter() and assigns it to each layer/module like this :
weights = nn.Parameter(torch.randn_like(self.encoder[0].weight))
self.encoder[0].weight.data = weights.clone()
self.decoder[0].weight.data = self.encoder[0].weight.data.transpose(0, 1)

This really confuses me, how is this sharing the same variable between these two layers?
Is it not just cloning the 'raw' data?
When I used this approach, and visualized the weights, I noticed the visualizations were different and that make me even more certain something is not right.
I'm not sure if the different visualizations were solely due to one being the transpose of the other one, or as I just already suspected, they are optimized independently (i.e. the weights are not shared between layers)
example weight initialization :

Output:
As it turns out, after further investigation, which was simply retransposing the decoder's weight and visualized it, they were indeed shared.
Below is the visualization for encoder and decoders weights :


Instruction: Masking: Mask everything after a specified token (eos)

Input:
My tgt tensor is in shape of [12, 32, 1] which is sequence_length, batch_size, token_idx.

What is the best way to create a mask which has ones for entries with <eos> and before in sequence, and zeros afterwards?

Currently I'm calculating my mask like this, which simply puts zeros where <blank> is, ones otherwise.

mask = torch.zeros_like(tgt).masked_scatter_((tgt != tgt_padding), torch.ones_like(tgt))

But the problem is, that my tgt can contain <blank> as well (before <eos>), in which cases I don't want to mask it out.

My temporary solution:

mask = torch.ones_like(tgt)
for eos_token in (tgt == tgt_eos).nonzero():
    mask[eos_token[0]+1:,eos_token[1]] = 0

Output:
I guess you are trying to create a mask for the PAD tokens. There are several ways. One of them is as follows.

# tensor is of shape [seq_len, batch_size, 1]
tensor = tensor.mul(tensor.ne(PAD).float())

Here, PAD stands for the index of the PAD_TOKEN. tensor.ne(PAD) will create a byte tensor where at PAD_TOKEN positions, 0 will be assigned and 1 elsewhere.

If you have examples like, "<s> I think <pad> so </s> <pad> <pad>". Then, I would suggest using different PAD tokens, for before and after </s>.

OR, if you have the length information for each sentence (in the above example, the sentence length is 6), then you can create the mask using the following function.

def sequence_mask(lengths, max_len=None):
    """
    Creates a boolean mask from sequence lengths.
    :param lengths: 1d tensor [batch_size]
    :param max_len: int
    """
    batch_size = lengths.numel()
    max_len = max_len or lengths.max()
    return (torch.arange(0, max_len, device=lengths.device)  # (0 for pad positions)
            .type_as(lengths)
            .repeat(batch_size, 1)
            .lt(lengths.unsqueeze(1)))


Instruction: torchtext data build_vocab / data_field

Input:
I want to ask you some about torchtext.

I have a task about abstractive text summarization, and I build a seq2seq model with pytorch.

I just wonder about data_field constructed by build_vocab function in torchtext.

In machine translation, i accept that two data_fields(input, output) are needed.

But, in summarization, input data and output data are same language.

Here, should I make two data_field(full_sentence, abstract_sentence) in here?

Or is it okay to use only one data_field?

I'm afraid that my wrong choice make model's performance down.

Please, give me a hint.

Output: You are right in the case of summarization and other tasks, it makes sense to build and use the same vocab for input and output


Instruction: How to convert torch tensor to pandas dataframe?

Input:
I'd like to convert a torch tensor to pandas dataframe but by using pd.DataFrame I'm getting a dataframe filled with tensors instead of numeric values.

import torch
import pandas as  pd
x = torch.rand(4,4)
px = pd.DataFrame(x)

Here's what I get when clicking on px in the variable explorer:

0   1   2   3
tensor(0.3880)  tensor(0.4598)  tensor(0.4239)  tensor(0.7376)
tensor(0.4174)  tensor(0.9581)  tensor(0.0987)  tensor(0.6359)
tensor(0.6199)  tensor(0.8235)  tensor(0.9947)  tensor(0.9679)
tensor(0.7164)  tensor(0.9270)  tensor(0.7853)  tensor(0.6921)

Output:
I found one possible way by converting torch first to numpy:
import torch
import pandas as  pd

x = torch.rand(4,4)
px = pd.DataFrame(x.numpy())


Instruction: Multiply rows of matrix by vector elementwise in pytorch?

Input:
I would like to do the below but using PyTorch.

The below example and description is from this post.

  I have a numeric matrix with 25 columns and 23 rows, and a vector of
  length 25. How can I multiply each row of the matrix by the vector
  without using a for loop?

  The result should be a 25x23 matrix (the same size as the input), but
  each row has been multiplied by the vector.

Example Code in R (source: reproducible example from @hatmatrix's answer):

matrix <- matrix(rep(1:3,each=5),nrow=3,ncol=5,byrow=TRUE)

     [,1] [,2] [,3] [,4] [,5]
[1,]    1    1    1    1    1
[2,]    2    2    2    2    2
[3,]    3    3    3    3    3

vector <- 1:5

Desired output:

     [,1] [,2] [,3] [,4] [,5]
[1,]    1    2    3    4    5
[2,]    2    4    6    8   10
[3,]    3    6    9   12   15

What is the best way of doing this using Pytorch?

Output:
The answer was so trivial that I overlooked it.
For simplicity I used a smaller vector and matrix in this answer.
Multiply rows of matrix by vector:
X = torch.tensor([[1,2,3],[5,6,7]])
y = torch.tensor([7,4])
X.transpose(0,1)*y
# or alternatively
y*X.transpose(0,1)

output:
tensor([[ 7, 20],
        [14, 24],
        [21, 28]])

tensor([[ 7, 20],
        [14, 24],
        [21, 28]])

Multiply columns of matrix by vector:
To multiply the columns of matrix by a vector you can use the same operator '*' but without the need to transpose the matrix (or vector) first
X = torch.tensor([[3, 5],[5, 5],[1, 0]])
y = torch.tensor([7,4])
X*y
# or alternatively
y*X

output:
tensor([[21, 20],
        [35, 20],
        [ 7,  0]])

tensor([[21, 20],
        [35, 20],
        [ 7,  0]])


Instruction: With pytorch DataLoader how to take in two ndarray (data & label)?

Input:
I have a training data features in ndarray of shape (100, 400, 3) as it's 100 images of 20x20 with RGB channel and label in shape (100, ). Do I need to combine them into one dataset or how can I pass it to Pytorch dataLoader in order to iterate over image and label later?

What I've tried so far

#turn ndarray of features and labels into tensors
transform = transforms.Compose([transforms.ToPILImage(),
                                transforms.ToTensor()])

Output:
As @Shai mentioned, DataLoader requires the input to be the Dataset class or its subclass. One of the simplest subclasses is TensorDataset and you can convert it from ndarray.

import torch
import numpy as np
import torch.utils as utils

train_x = torch.Tensor(np.random.randn(100,400,3))
train_y = torch.Tensor(np.random.randint(0,2,100))

dataset = utils.data.TensorDataset(train_x, train_y)
dataloader = utils.data.DataLoader(dataset)


Instruction: pytorch cannot install using anaconda prompt

Input:
C:\Users\User>pip install pytorch

When I run this command anaconda prompt display this error.

Collecting pytorch
  Downloading https://files.pythonhosted.org/packages/ee/67/f403d4ae6e9cd74b546ee88cccdb29b8415a9c1b3d80aebeb20c9ea91d96/pytorch-1.0.2.tar.gz
Building wheels for collected packages: pytorch
  Building wheel for pytorch (setup.py) ... error
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\User\AppData\Local\Temp\pip-wheel-lqbohuk2' --python-tag cp37:
  ERROR: Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\setup.py", line 15, in <module>
      raise Exception(message)
  Exception: You tried to install "pytorch". The package named for PyTorch is "torch"
  ----------------------------------------
  ERROR: Failed building wheel for pytorch
  Running setup.py clean for pytorch
Failed to build pytorch
Installing collected packages: pytorch
  Running setup.py install for pytorch ... error
    ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-oah_3fo2\install-record.txt' --single-version-externally-managed --compile:
    ERROR: Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\setup.py", line 11, in <module>
        raise Exception(message)
    Exception: You tried to install "pytorch". The package named for PyTorch is "torch"
    ----------------------------------------
ERROR: Command "'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-yto7o19p\\pytorch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-oah_3fo2\install-record.txt' --single-version-externally-managed --compile" failed with error code 1 in C:\Users\User\AppData\Local\Temp\pip-install-yto7o19p\pytorch\

C:\Users\User>pip install torch

When I tried to install torch it gives me this error message

Collecting torch
  Downloading https://files.pythonhosted.org/packages/f8/02/880b468bd382dc79896eaecbeb8ce95e9c4b99a24902874a2cef0b562cea/torch-0.1.2.post2.tar.gz (128kB)
     |████████████████████████████████| 133kB 544kB/s
Requirement already satisfied: pyyaml in c:\users\user\anaconda3\lib\site-packages (from torch) (5.1.1)
Building wheels for collected packages: torch
  Building wheel for torch (setup.py) ... error
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\User\AppData\Local\Temp\pip-wheel-64iazehi' --python-tag cp37:
  ERROR: running bdist_wheel
  running build
  running build_deps
  Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 265, in <module>
      description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
    File "C:\Users\User\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
      return distutils.core.setup(**attrs)
    File "C:\Users\User\Anaconda3\lib\distutils\core.py", line 148, in setup
      dist.run_commands()
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
      self.run_command(cmd)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\Anaconda3\lib\site-packages\wheel\bdist_wheel.py", line 192, in run
      self.run_command('build')
    File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\Anaconda3\lib\distutils\command\build.py", line 135, in run
      self.run_command(cmd_name)
    File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 51, in run
      from tools.nnwrap import generate_wrappers as generate_nn_wrappers
  ModuleNotFoundError: No module named 'tools.nnwrap'
  ----------------------------------------
  ERROR: Failed building wheel for torch
  Running setup.py clean for torch
  ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' clean --all:
  ERROR: running clean
  error: [Errno 2] No such file or directory: '.gitignore'
  ----------------------------------------
  ERROR: Failed cleaning build dir for torch
Failed to build torch
Installing collected packages: torch
  Running setup.py install for torch ... error
    ERROR: Complete output from command 'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-en00_2m3\install-record.txt' --single-version-externally-managed --compile:
    ERROR: running install
    running build_deps
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 265, in <module>
        description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
      File "C:\Users\User\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
        return distutils.core.setup(**attrs)
      File "C:\Users\User\Anaconda3\lib\distutils\core.py", line 148, in setup
        dist.run_commands()
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
        self.run_command(cmd)
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 99, in run
        self.run_command('build_deps')
      File "C:\Users\User\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
        self.distribution.run_command(command)
      File "C:\Users\User\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\setup.py", line 51, in run
        from tools.nnwrap import generate_wrappers as generate_nn_wrappers
    ModuleNotFoundError: No module named 'tools.nnwrap'
    ----------------------------------------
ERROR: Command "'C:\Users\User\Anaconda3\python.exe' -u -c 'import setuptools, tokenize;__file__='"'"'C:\\Users\\User\\AppData\\Local\\Temp\\pip-install-5hkp2cz9\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\User\AppData\Local\Temp\pip-record-en00_2m3\install-record.txt' --single-version-externally-managed --compile" failed with error code 1 in C:\Users\User\AppData\Local\Temp\pip-install-5hkp2cz9\torch\

Other packsges can install by using pip install command but pytorch and torch didn't work like that

Output: conda install PyTorch -c PyTorch is fine no need to run pip3 install torchvision


Instruction: Do I need to match the representation of my output layer and the target labels in PyTorch Nllloss?

Input:
I'm trying to solve a classification problem with 3 possible outputs: 0, 1, or 2.
My output layer finally outputs a vector of probabilities for each label, say [0.3,0.4,0.3]

My loss function is defined such:

loss = criterion(output_batch, label_batch) #criterion = nn.NLLLoss()

Now my question has to do with the outputs and labels not matching in the way the store data. The output is in the form of a size=3 probability vector (adding to 1 using soft max), and my target labels are simple scalars.

I can convert my labels to vectors when the loss function is calculated but I'm not sure if this is necessary

0 ==> [1,0,0]
1 ==> [0,1,0]
2 ==> [0,0,1]

Can someone please shed light on this issue? Thanks!

Output:
Let's say your classes are: cat, dog and capibara.

You have so called softmax predictions.

[0.3,0.4,0.3]

The softmax function is pumping one result at a top. In this case if dog is under 0.4 our output is predicting the dog.

Note how the predictions sum to 1 = 0.3+0.4+0.3.

Now you need to calculate the log of that which is log softmax, and then NLL is just negative of that.

  I can convert my labels to vectors when the loss function is calculated but I'm not sure if this is necessary?

0 ==> [1,0,0]
1 ==> [0,1,0]
2 ==> [0,0,1]

This is not necessary in your case. This means we had three different estimations (bs=3) while you showed just one.

Here is a little exercise:

batch_size, n_classes = 10, 5
x = torch.randn(batch_size, n_classes)
print("x:",x)

target = torch.randint(n_classes, size=(batch_size,), dtype=torch.long)
print("target:",target)

def log_softmax(x):
    return x - x.exp().sum(-1).log().unsqueeze(-1)

def nll_loss(p, target):
    return -p[range(target.shape[0]), target].mean()

pred = log_softmax(x)
print ("pred:", pred)
ohe = torch.zeros(batch_size, n_classes)
ohe[range(ohe.shape[0]), target]=1
print("ohe:",ohe)

pe = pred[range(target.shape[0]), target]
print("pe:",pe)

mean = pred[range(target.shape[0]), target].mean()
print("mean:",mean)

negmean = -mean
print("negmean:", negmean)

loss = nll_loss(pred, target)
print("loss:",loss)

Out:

x: tensor([[ 1.5837, -1.3132,  1.5513,  1.4422,  0.8072],
        [ 1.1740,  1.9250,  0.4258, -1.0320, -0.4650],
        [-1.2447, -0.5360, -1.4950,  1.2020,  1.2724],
        [ 0.2300,  0.2587, -0.4463, -0.1397, -0.3617],
        [-0.7983,  0.7742,  0.0035,  0.9963, -0.7926],
        [ 0.7575, -0.8008,  0.7995,  0.0448,  0.6621],
        [-1.7153,  0.7672, -0.6841, -0.4826, -0.8614],
        [ 0.0263,  0.7244,  0.8751, -1.0226, -1.3762],
        [ 0.0192, -0.4368, -0.4010, -1.0660,  0.0364],
        [-0.5120, -1.4871,  0.6758,  1.2975,  0.2879]])
target: tensor([0, 4, 3, 0, 0, 4, 1, 2, 4, 2])
pred: tensor([[-1.2094, -4.1063, -1.2418, -1.3509, -1.9859],
        [-1.3601, -0.6091, -2.1083, -3.5661, -2.9991],
        [-3.3233, -2.6146, -3.5736, -0.8766, -0.8063],
        [-1.3302, -1.3015, -2.0065, -1.7000, -1.9220],
        [-2.7128, -1.1403, -1.9109, -0.9181, -2.7070],
        [-1.2955, -2.8538, -1.2535, -2.0081, -1.3909],
        [-3.0705, -0.5881, -2.0394, -1.8379, -2.2167],
        [-1.7823, -1.0841, -0.9334, -2.8311, -3.1847],
        [-1.2936, -1.7496, -1.7138, -2.3788, -1.2764],
        [-2.5641, -3.5393, -1.3764, -0.7546, -1.7643]])
ohe: tensor([[1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 0., 0., 1., 0.],
        [1., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 1., 0., 0., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 0., 1.],
        [0., 0., 1., 0., 0.]])
pe: tensor([-1.2094, -2.9991, -0.8766, -1.3302, -2.7128, -1.3909, -0.5881, -0.9334,
        -1.2764, -1.3764])
mean: tensor(-1.4693)
negmean: tensor(1.4693)
loss: tensor(1.4693)


Instruction: how to use RGB values in feedforward neural network?

Input:
I have data set of colored images in the form of ndarray (100, 20, 20, 3) and 100 corresponding labels. When passing them as input to a fully connected neural network (not CNN), what should I do with the 3 values of RGB? Average them perhaps lose some information, but if not manipulating them, my main issue is batch size, as demo-ed below in pytorch.

for epoch in range(n_epochs):
    for i, (images, labels) in enumerate(train_loader):
        # because of rgb values, now images is 3 times the length of labels
        images = Variable(images.view(-1, 400))
        labels = Variable(labels)
        optimizer.zero_grad()
        outputs = net(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

This returns 'ValueError: Expected input batch_size (300) to match target batch_size (100).' Should I have reshaped images into (1, 1200) dimension tensors? Thanks in advance for answers.

Output:
Since size of labels is (100,), so your batch data should be with shape of (100, H, W, C). I'm assuming your data loader is returning a tensor with shape of (100,20,20,3). The error happens because you reshape the tensor to (300,400).

Check your network architecture whether the input tensor shape is (20,20,3).
If your network can only accept single channel images, you can first convert your RGB to grayscale images.
Or, modify your network architecture to make it accept 3 channels images. One convenient way is adding an extra layer reducing 3 channels to 1 channel, and you do not need to change the other parts of the network.


Instruction: Unexpected import behavior: non-existing module imported

Input:
I have a torch extension (but I think the error I get is independent of torch), let's call it foo, that is built with setuptools. However, it depends on whether there is a GPU available in the system, in which case some CUDA code is compiled:

from torch.utils.cpp_extension import CppExtension, BuildExtension, CUDAExtension

if cuda_available:
    ext = CUDAExtension(
            'foo_gpu',
            prefixed(prefix, ['foo.cpp', 'foo_kernel.cu']),
            define_macros=[('COMPILE_CUDA', '1')])
else:
    ext = CppExtension(
            'foo_cpu',
            prefixed(prefix, ['foo.cpp']))

setup(name=ext.name,
      version='1.0.0',
      ext_modules=[ext_module],
      extra_compile_args=['-mmacosx-version-min=10.9'],
      cmdclass={'build_ext': BuildExtension})

Notice how the module is compiled to be either foo_gpu or foo_cpu.

Later, I import it as follows:

try:
    import foo_gpu as foo_backend
    CUDA_SUPPORTED = True
except ImportError:
    CUDA_SUPPORTED = False
    # Try importing the cpu version
    try:
        import foo_cpu as foo_backend
    except ImportError:
        ...

Let's say I compile it with CUDA support, so foo_gpu goes into PIP (actually goes into PIP as foo-gpu, not sure why?)

Now, I uninstalled it, pip uninstall foo-gpu, and compile it with CPU support only, and pip shows foo-cpu 1.0.0.

BUT, now, when I run the import code above, it still finds foo-gpu, i.e., the first import statement succeeds! Even though it does not show up in pip.

EDIT. I checked sys.path and found there is one folder that contains something with gpu in my conda env:

$ ls ~/miniconda3/envs/cpu_env/lib/python3.7/site-packages/foo_cpu-1.0.0-py3.7-linux-x86_64.egg/
 EGG-INFO
 __pycache__
 foo_cpu.cpython-37m-x86_64-linux-gnu.so
 foo_cpu.py
 foo_gpu.cpython-37m-x86_64-linux-gnu.so

But how should it get there? In this env (cpu_env), I never compiled with GPU support. Are there some caches that get invoked?

Output: Turns out the GPU build in was cached by setuptools (build/, dist/, etc.) and also installed!


Instruction: How do I reshape this tensor?

Input:
I have a torch.tensor that looks like this:

tensor([[[A,B,C],
         [D,E,F],
         [G,H,I]],

        [[J,K,L],
         [M,N,O],
         [P,Q,R]]]

I want to reshape this tensor so that its dimensions are (18, 1). I want the new tensor to look like this:

tensor([A,
        J,
        B,
        K,
        C,
        L,
        D,
        M,
       ...
        I,
        R])

I've tried tensor.view(-1,1) but this doesn't work..

Output:
a = torch.arange(18).view(2,3,3)

print(a)
#tensor([[[ 0,  1,  2],
#         [ 3,  4,  5],
#         [ 6,  7,  8]],
#
#        [[ 9, 10, 11],
#         [12, 13, 14],
#         [15, 16, 17]]])

aa = a.permute(1,2,0).flatten()

print(aa)
#tensor([ 0,  9,  1, 10,  2, 11,  3, 12,  4, 13,  5, 14,  6, 15,  7, 16,  8, 17])


Instruction: OverflowError: Python int too large to convert to C long torchtext.datasets.text_classification.DATASETS['AG_NEWS']()

Input:
I have 64 bit windows 10 OS
I have installed python 3.6.8
I have installed torch and torchtext using pip.
torch version is 1.2.0

I am trying to load AG_NEWS dataset using below code:

import torch
import torchtext
from torchtext.datasets import text_classification
NGRAMS = 2
import os
if not os.path.isdir('./.data'):
    os.mkdir('./.data')
train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS'](root='./.data', ngrams=NGRAMS, vocab=None)

On the last statement of above code, I am getting below error:

---------------------------------------------------------------------------
OverflowError                             Traceback (most recent call last)
<ipython-input-1-7e8544fdaaf6> in <module>
      6 if not os.path.isdir('./.data'):
      7     os.mkdir('./.data')
----> 8 train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS'](root='./.data', ngrams=NGRAMS, vocab=None)
      9 # BATCH_SIZE = 16
     10 # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

c:\users\pramodp\appdata\local\programs\python\python36\lib\site-packages\torchtext\datasets\text_classification.py in AG_NEWS(*args, **kwargs)
    168     """
    169
--> 170     return _setup_datasets(*(("AG_NEWS",) + args), **kwargs)
    171
    172

c:\users\pramodp\appdata\local\programs\python\python36\lib\site-packages\torchtext\datasets\text_classification.py in _setup_datasets(dataset_name, root, ngrams, vocab, include_unk)
    126     if vocab is None:
    127         logging.info('Building Vocab based on {}'.format(train_csv_path))
--> 128         vocab = build_vocab_from_iterator(_csv_iterator(train_csv_path, ngrams))
    129     else:
    130         if not isinstance(vocab, Vocab):

c:\users\pramodp\appdata\local\programs\python\python36\lib\site-packages\torchtext\vocab.py in build_vocab_from_iterator(iterator)
    555     counter = Counter()
    556     with tqdm(unit_scale=0, unit='lines') as t:
--> 557         for tokens in iterator:
    558             counter.update(tokens)
    559             t.update(1)

c:\users\pramodp\appdata\local\programs\python\python36\lib\site-packages\torchtext\datasets\text_classification.py in _csv_iterator(data_path, ngrams, yield_cls)
     33     with io.open(data_path, encoding="utf8") as f:
     34         reader = unicode_csv_reader(f)
---> 35         for row in reader:
     36             tokens = ' '.join(row[1:])
     37             tokens = tokenizer(tokens)

c:\users\pramodp\appdata\local\programs\python\python36\lib\site-packages\torchtext\utils.py in unicode_csv_reader(unicode_csv_data, **kwargs)
    128             maxInt = int(maxInt / 10)
    129
--> 130     csv.field_size_limit(sys.maxsize)
    131
    132     if six.PY2:

OverflowError: Python int too large to convert to C long

I think the issue is with either windows os or torchtext because I am getting same error for below code as well.

pos = data.TabularDataset( path='data/pos/pos_wsj_train.tsv', format='tsv', fields=[('text', data.Field()),
        ('labels', data.Field())])

Can somebody please help? and mainly I don't have any large numerical values in the file.

Output:
I also encountered a similar problem. I changed a line of code in my torchtext\utils.py file and my error disappeared.
Changed this:
csv.field_size_limit(sys.maxsize)

To this:
csv.field_size_limit(maxInt)