Instruction: LSTM implementation / overfitting

Input:
I am having a problem on an implementation of LSTM. I am not sure if I have the right implementation or this is just an overfitting problem. I am doing essay grading using a LSTM, scoring text with score from 0 - 10 (or other range of score). I am using the ASAP kaggle competition data as one of the training data.

However, the main goal is to achieve good performance on a private dataset, with around 500 samples. The 500 samples includes validation and training set. I have previously done some experiment and got the model to work, but after fiddling with something, the model doesn't fit anymore. The model does not improve at all. I have also re-implemented the code in a cleaner manner with much more obejct oriented code and still can't reproduce my previous result.

However, I am getting the model to fit to my data, just there is tremendous overfitting. I am not sure if this is an implementation problem of some sort or just overfitting, but I cannot get the model to work. The maximum I can get it to is 0.35 kappa using LSTM on the ASAP data essay set 1. For some bizarre reason, I can get a single layer fully connected model to have 0.75 kappa. I think this is an implementation problem but I am not sure.

Here is my old code:

train.py

import gensim
import numpy as np
import pandas as pd
import torch
from sklearn.metrics import cohen_kappa_score
from torch import nn
import torch.utils.data as data_utils
from torch.optim import Adam

from dataset import AESDataset
from network import Network

from optimizer import Ranger
from qwk import quadratic_weighted_kappa, kappa

batch_size = 32

device = "cuda:0"
torch.manual_seed(1000)
# Load data from csv
file_name = "data/data_new.csv"
data = pd.read_csv(file_name)
arr = data.to_numpy()
text = arr[:, :2]
text = [str(line[0]) + str(line[1]) for line in text]
text = [gensim.utils.simple_preprocess(line) for line in text]

score = arr[:,2]

score = [sco*6 for sco in score]
score = np.asarray(score, dtype=int)

train_dataset = AESDataset(text_arr=text[:400], scores=score[:400])
test_dataset = AESDataset(text_arr=text[400:], scores=score[400:])

score = torch.tensor(score).view(-1,1).long().to(device)

train_loader = data_utils.DataLoader(train_dataset,shuffle=True, batch_size=batch_size, drop_last=True)
test_loader = data_utils.DataLoader(test_dataset,shuffle=True,batch_size=batch_size, drop_last=True)

out_class = 61

epochs = 1000

model = Network(out_class).to(device)
model.load_state_dict(torch.load("model/best_model"))
y_onehot = torch.FloatTensor(batch_size, out_class).to(device)
optimizer = Adam(model.parameters())
criti = torch.nn.CrossEntropyLoss()
# model, optimizer = amp.initialize(model, optimizer, opt_level="O2")

step = 0

for i in range(epochs):
    #Testing
    if i % 1 == 0:
        total_loss = 0
        total_kappa = 0
        total_batches = 0
        model.eval()
        for (text, score) in test_loader:

            out = model(text)
            out_score = torch.argmax(out, 1)
            y_onehot.zero_()
            y_onehot.scatter_(1, score, 1)
            kappa_l = cohen_kappa_score(score.view(batch_size).tolist(), out_score.view(batch_size).tolist())
            score = score.view(-1)
            loss = criti(out, score.view(-1))
            total_loss += loss
            total_kappa += kappa_l
            total_batches += 1
        print(f"Epoch {i} Testing kappa {total_kappa/total_batches} loss {total_loss/total_batches}")
        with open(f"model/epoch_{i}", "wb") as f:
            torch.save(model.state_dict(),f)
        model.train()
    #Training

    for (text, score) in train_loader:

        optimizer.zero_grad()
        step += 1
        out = model(text)
        out_score = torch.argmax(out,1)
        y_onehot.zero_()
        y_onehot.scatter_(1, score, 1)
        kappa_l = cohen_kappa_score(score.view(batch_size).tolist(),out_score.view(batch_size).tolist())
        loss = criti(out, score.view(-1))
        print(f"Epoch {i} step {step} kappa {kappa_l} loss {loss}")
        loss.backward()
        optimizer.step()

dataset.py

import gensim
import torch
import numpy as np

class AESDataset(torch.utils.data.Dataset):
    def __init__(self, text_arr, scores):
        self.data = text_arr
        self.scores = scores
        self.w2v_model = ("w2vec_model_all")
        self.max_len = 500
    def __getitem__(self, item):
        vector = []
        essay = self.data[item]

        pad_vec = [1 for i in range(300)]
        for i in range(self.max_len - len(essay)):
            vector.append(pad_vec)
        for word in essay:
            word_vec = pad_vec
            try:
                word_vec = self.w2v_model[word]
            except:
                #print(f"Skipping word as word {word} not in dictionary")
                word_vec = pad_vec

            vector.append(word_vec)
        #print(len(vector))
        vector = np.stack(vector)
        tensor = torch.tensor(vector[:self.max_len]).float().to("cuda")
        score = self.scores[item]
        score = torch.tensor(score).long().to("cuda").view(1)

        return tensor, score

    def __len__(self):
        return len(self.scores)

network.py

import torch.nn as nn
import torch

import torch.nn.functional as F

class Network(nn.Module):
    def __init__(self, output_size):
        super(Network, self).__init__()
        self.lstm = nn.LSTM(300,500,1, batch_first=True)
        self.dropout = nn.Dropout(p=0.5)
        #self.l2 = nn.L2
        self.linear = nn.Linear(500,output_size)

    def forward(self,x):
        x, _ = self.lstm(x)
        x = x[:,-1,:]
        x = self.dropout(x)
        x = self.linear(x)

        return x

My new code: https://github.com/Clement-Hui/EssayGrading

Output:
I think the problem is in the training code since you are using LSTM you are supposed to flush down the hidden and cell state after every epoch and detach it from the computation graph after each batch.

network.py

import torch.nn as nn
import torch

import torch.nn.functional as F

class Network(nn.Module):
    def __init__(self, output_size):
        super(Network, self).__init__()
        self.lstm = nn.LSTM(300,500,1, batch_first=True)
        self.dropout = nn.Dropout(p=0.5)
        #self.l2 = nn.L2
        self.linear = nn.Linear(500,output_size)

    def forward(self,x,hidden):
        x, hidden = self.lstm(x,hidden)
        x = x.contiguous().view(-1, 500)
        x = self.dropout(x)
        x = self.linear(x)
        return x , hidden

    def init_hidden(self,batch_size):
        weights = next(self.parameters()).data
        hidden = (weights.new(1 , batch_size,500).zero_().cuda(),
                  weights.new(1 , batch_size,500).zero_().cuda())
        return hidden

train.py

# your code for intializing the model and data and all other stuff
for i in range(epochs):

    #Testing
    if i % 1 == 0:
        total_loss = 0
        total_kappa = 0
        total_batches = 0
        model.eval()
        val_h  = model.init_hidden(batch_size) # intialize the hidden state
        for (text, score) in test_loader:
           # Creating new variables for the hidden state, otherwise
           # we'd backprop through the entire training history
            val_h = tuple([each.data for each in val_h])
            out ,  val_h  = model(text,val_h)
            out_score = torch.argmax(out, 1)
            y_onehot.zero_()
            y_onehot.scatter_(1, score, 1)
            kappa_l = cohen_kappa_score(score.view(batch_size).tolist(), out_score.view(batch_size).tolist())
            score = score.view(-1)
            loss = criti(out, score.view(-1))
            total_loss += loss
            total_kappa += kappa_l
            total_batches += 1
        print(f"Epoch {i} Testing kappa {total_kappa/total_batches} loss {total_loss/total_batches}")
        with open(f"model/epoch_{i}", "wb") as f:
            torch.save(model.state_dict(),f)
    model.train()

    #Training
    h =  model.init_hidden(batch_size) # intialize the hidden state
    for (text, score) in train_loader:
        optimizer.zero_grad()
        step += 1
        # Creating new variables for the hidden state, otherwise
        # we'd backprop through the entire training history
        h = tuple([each.data for each in h])
        out , h  = model(text,h)
        out_score = torch.argmax(out,1)
        y_onehot.zero_()
        y_onehot.scatter_(1, score, 1)
        kappa_l = cohen_kappa_score(score.view(batch_size).tolist(),out_score.view(batch_size).tolist())
        loss = criti(out, score.view(-1))
        print(f"Epoch {i} step {step} kappa {kappa_l} loss {loss}")
        loss.backward()
        optimizer.step()

Do let me know if the changes mentioned works or not.


Instruction: The location of source code for torch.mean()

Input:
I'm trying to find the source code for torch.mean and am unable to find it in the pytorch github. It is under math operations but I can't find it at all.

I've looked everywhere and inspected most pages under pytorch/torch and am still unable to find it.

I even did ?? in a jupyter notebook but it just returned a useless docstring

Output:
Since operations components are written in C++, they are not callable with operations such as ?? or "__file__" or "getsourcefile" type of operations.

The files appear to be here, written in C++:

https://github.com/pytorch/pytorch/blob/master/caffe2/operators/mean_op.cc
https://github.com/pytorch/pytorch/blob/master/caffe2/operators/mean_op.h


Instruction: Hidden state tensors have a different order than the returned tensors

Input:
As part of GRU training, I want to retrieve the hidden state tensors.

I have defined a GRU with two layers:

self.lstm = nn.GRU(params.vid_embedding_dim, params.hidden_dim , 2)

The forward function is defined as follows (the following is just a part of the implementation):

    def forward(self, s, order, batch_size, where, anchor_is_phrase = False):
    """
    Forward prop.
    """
      # s is of shape [128 , 1 , 300] , 128 is batch size
      output, (a,b) = self.lstm(s.cuda())
      output.data.contiguous()

And out is of shape: [128 , 400] (128 is the number of samples which each one is embedded in 400 dimensional vector).

I understand that out is the output of the last hidden state and thus I expect it to be equal to b. However, after I checked the values I saw that it's indeed equal but b contains the tensor in a different order, that is for example output[0] is b[49]. Am I missing something here ?

Thanks.

Output:
I understand your confusion. Have a look the example bellow and the comments:

# [Batch size, Sequence length, Embedding size]
inputs = torch.rand(128, 5, 300)
gru = nn.GRU(input_size=300, hidden_size=400, num_layers=2, batch_first=True)

with torch.no_grad():
    # output is all hidden states, for each element in the batch of the last layer in the RNN
    # a is the last hidden state of the first layer
    # b is the last hidden state of the second (last) layer
    output, (a, b) = gru(inputs)

If we print out the shapes, they will confirm our understanding:

print(output.shape) # torch.Size([128, 5, 400])
print(a.shape) # torch.Size([128, 400])
print(b.shape) # torch.Size([128, 400])

Also, we can test whether the last hidden state, for each element in the batch, of the last layer, obtained from output is equal to b:

np.testing.assert_almost_equal(b.numpy(), output[:,:-1,:].numpy())

Finally, we can create an RNN with 3 layers, and run the same tests:

gru = nn.GRU(input_size=300, hidden_size=400, num_layers=3, batch_first=True)
with torch.no_grad():
    output, (a, b, c) = gru(inputs)

np.testing.assert_almost_equal(c.numpy(), output[:,-1,:].numpy())

Again, the assertion passes but only if we do it for c, which is now the last layer of the RNN. Otherwise:

np.testing.assert_almost_equal(b.numpy(), output[:,-1,:].numpy())

Raises an error:

  AssertionError:  Arrays are not almost equal to 7 decimals

I hope that this makes things clear for you.


Instruction: Pytorch: Understand how nn.Module class internally work

Input:
Generally, a nn.Module can be inherited by a subclass as below.

def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform(m.weight)  #

class LinearRegression(nn.Module):
    def __init__(self):
        super(LinearRegression, self).__init__()
        self.fc1 = nn.Linear(20, 1)
        self.apply(init_weights)

    def forward(self, x):
        x = self.fc1(x)
        return x

My 1st question is, why I can simply run the code below even my __init__ doesn't have any positinoal arguments for training_signals and it looks like that training_signals is passed to forward() method. How does it work?

model = LinearRegression()
training_signals = torch.rand(1000,20)
model(training_signals)

The second question is that how does self.apply(init_weights) internally work? Is it executed before calling forward method?

Output:
  Q1: Why I can simply run the code below even my __init__ doesn't have any positional arguments for training_signals and it looks like that training_signals is passed to forward() method. How does it work?

First, the __init__ is called when you run this line:

model = LinearRegression()

As you can see, you pass no parameters, and you shouldn't. The signature of your __init__ is the same as the one of the base class (which you call when you run super(LinearRegression, self).__init__()). As you can see here, nn.Module's init signature is simply def __init__(self) (just like yours).

Second, model is now an object. When you run the line below:

model(training_signals)

You are actually calling the __call__ method and passing training_signals as a positional parameter. As you can see here, among many other things, the __call__ method calls the forward method:

result = self.forward(*input, **kwargs)

passing all parameters (positional and named) of the __call__ to the forward.

  Q2: How does self.apply(init_weights) internally work? Is it executed before calling forward method?

PyTorch is Open Source, so you can simply go to the source-code and check it. As you can see here, the implementation is quite simple:

def apply(self, fn):
    for module in self.children():
        module.apply(fn)
    fn(self)
    return self

Quoting the documentation of the function: it "applies fn recursively to every submodule (as returned by .children()) as well as self". Based on the implementation, you can also understand the requirements:

fn must be a callable;
fn receives as input only a Module object;


Instruction: Expected object of device type cuda but got device type cpu in Pytorch

Input:
I have the following code which calculates a loss function:

class MSE_loss(nn.Module):
    """
    : metric: L1, L2 norms or cosine similarity
    : mode: training or evaluation mode
    """

    def __init__(self,metric, mode, weighted_sum = False):
        super(MSE_loss, self).__init__()
        self.metric = metric.lower()
        self.loss_function = nn.MSELoss()
        self.mode = mode.lower()
        self.weighted_sum = weighted_sum

    def forward(self, output1, output2, labels):
        self.labels = labels
        self.linear = nn.Linear(output1.size()[0],1)

        if self.metric == 'cos':
            self.d= F.cosine_similarity(output1, output2)
        elif self.metric == 'l1':
            self.d = torch.abs(output1-output2)
        elif self.metric == 'l2':
            self.d = torch.sqrt((output1-output2)**2)

        def dimensional_reduction(forward):
            if self.weighted_sum:
                distance = self.linear(self.d)
            else:
                distance = torch.mean(self.d,1)
            return distance

        def estimate_loss(forward):
            distance = dimensional_reduction(self.d)
            pred = torch.exp(-distance)
            pred = torch.round(pred)
            loss = self.loss_function(pred, self.labels)
            return pred, loss

        pred, loss = estimate_loss(self.d)

        if self.mode == 'training':
            return loss
        else:
            return pred, loss

Given

criterion = MSE_loss('l1','training', weighted_sum = True)

I would like to get the distance after having gone through the self.linear neuron when implementing criterion. I am, however, prompted with the error 'Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_addmm' indicating that something is wrong. I have omitted the first part of the code, but I provide the whole error message, so that you can get an idea of what is going on.

RuntimeError                              Traceback (most recent call last)
<ipython-input-253-781ed4791260> in <module>()
      7 criterion = MSE_loss('l1','training', weighted_sum = True)
      8
----> 9 train(test_net, train_loader, 10, batch_size, optimiser, clip, criterion)

<ipython-input-207-02fecbfe3b1c> in train(SNN, dataloader, epochs, batch_size, optimiser, clip, criterion)
     57
     58             # calculate the loss and perform backprop
---> 59             loss = criterion(output1, output2, labels)
     60             a = [[n,p, p.grad] for n,p in SNN.named_parameters()]
     61

~/.conda/envs/dalkeCourse/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

<ipython-input-248-fb88b987ce71> in forward(self, output1, output2, labels)
     49             return pred, loss
     50
---> 51         pred, loss = estimate_loss(self.d)
     52
     53         if self.mode == 'training':

<ipython-input-248-fb88b987ce71> in estimate_loss(forward)
     43
     44         def estimate_loss(forward):
---> 45             distance = dimensional_reduction(self.d)
     46             pred = torch.exp(-distance)
     47             pred = torch.round(pred)

<ipython-input-248-fb88b987ce71> in dimensional_reduction(forward)
     36             else:
     37                 if self.weighted_sum:
---> 38                     self.d = self.linear(self.d)
     39                 else:
     40                     self.d = torch.mean(self.d,1)

~/.conda/envs/dalkeCourse/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

~/.conda/envs/dalkeCourse/lib/python3.6/site-packages/torch/nn/modules/linear.py in forward(self, input)
     85
     86     def forward(self, input):
---> 87         return F.linear(input, self.weight, self.bias)
     88
     89     def extra_repr(self):

~/.conda/envs/dalkeCourse/lib/python3.6/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1368     if input.dim() == 2 and bias is not None:
   1369         # fused op is marginally faster
-> 1370         ret = torch.addmm(bias, input, weight.t())
   1371     else:
   1372         output = input.matmul(weight.t())

RuntimeError: Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_addmm

self.d is however a tensor, but this has already been passed into GPU, as shown below:

self.d =
tensor([[3.7307e-04, 8.4476e-04, 4.0426e-04,  ..., 4.2015e-04, 1.7830e-04,
         1.2833e-04],
        [3.9271e-04, 4.8325e-04, 9.5238e-04,  ..., 1.5126e-04, 1.3420e-04,
         3.9260e-04],
        [1.9278e-04, 2.6530e-04, 8.6903e-04,  ..., 1.6985e-05, 9.5103e-05,
         1.9610e-04],
        ...,
        [1.8257e-05, 3.1304e-04, 4.6398e-04,  ..., 2.7327e-04, 1.1909e-04,
         1.5069e-04],
        [1.7577e-04, 3.4820e-05, 9.4168e-04,  ..., 3.2848e-04, 2.2514e-04,
         5.4275e-05],
        [4.2916e-04, 1.6155e-04, 9.3186e-04,  ..., 1.0950e-04, 2.5083e-04,
         3.7374e-06]], device='cuda:0', grad_fn=<AbsBackward>)

Output:
In the forward of your MSE_loss, you define a linear layer that is probably still in the CPU (you didn't provide an MCVE, so I can only assume):

self.linear = nn.Linear(output1.size()[0], 1)

If you want to try and see if this is the problem, you can:

self.linear = nn.Linear(output1.size()[0], 1).cuda()

However, if self.d is in the CPU, then it would fail again. To solve this, you could move the linear to the same device of the self.d tensor by doing this:

def forward(self, output1, output2, labels):
    self.labels = labels
    self.linear = nn.Linear(output1.size()[0], 1)

    if self.metric == 'cos':
        self.d = F.cosine_similarity(output1, output2)
    elif self.metric == 'l1':
        self.d = torch.abs(output1-output2)
    elif self.metric == 'l2':
        self.d = torch.sqrt((output1-output2)**2)

    # move self.linear to the correct device
    self.linear = self.linear.to(self.d.device)


Instruction: Passing word2vec embedding to a custom LSTM pytorch model

Input:
I have a set of input sentences. I am using the pretrained word2vec model from gensim to get the embedding of the input sentences. I want to pass these embeddings as input to a custom pytorch LSTM model

hidden_size = 32
num_layers = 1
num_classes = 2

class customModel(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_classes):
        super(customModel, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.bilstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=False, bidirectional=True)
        self.fcl = nn.Linear(hidden_size*2, num_classes)

    def forward(self, x):
        # Set initial hidden and cell states
        h0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device)
        c0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device)

        # Forward propagate LSTM
        out, hidden = self.bilstm(x, (h0, c0))
        fw_bilstm = out[-1, :, :self.hidden_size]
        bk_bilstm = out[0, :, :self.hidden_size]
        concat_fw_bw = torch.cat((fw_bilstm, bk_bilstm), dim = 1)
        fc = self.fcl(concat_fw_bw)
        x = F.softmax(F.relu(fc))
        return x

Now I initialize the model object.

model = customModel(300, hidden_size, num_layers, num_classes)

Get embedding for the input sentences

sentences = [['my', 'name', 'is', 'nad'], ['i', 'love', 'nlp', 'proc']]
embedding = create_embedding(sentences)
embedding_torch = torch.FloatTensor(embedding)

Now I want to pass these embeddings to the model to get the prediction

for item in embedding_torch:
    item = item.view((1, item.size()[0], item.size()[1]))
    for epoch in range(1):
        tag_scores = model(item)
        print (tag_scores)

Which throws me runtime error

RuntimeError: Expected hidden[0] size (2, 4, 32), got (2, 1, 32)

I am not sure why this is happening. My understanding is h0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device) line is calculating the hidden dimension properly.

What am I missing? Please suggest.

Output:
The backbone of your model is nn.LSTM which expects inputs with size [sequence_length, batch_size, embedding_size]. On the other hand, the inputs you are providing the model have size [1, sequence_lenth, embedding_size]. What I would do is create the nn.LSTM as:

# With batch_first=True
self.bilstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, bidirectional=True)

That way, the model would expect the inputs to be of size [batch_size, sequence_length, embedding_size]. Then, instead of going through each element in the batch separately, do:

tag_scores = model(embedding_torch)


Instruction: Calculate the shape in fully connected layer after cnn

Input:
I am trying to learn PyTorch. But I am really confused about the shape in a fully connected layer after convolution and max pooling.

Case 1. How we calculate the number 5408 in nn.Linear
I think 5408 = 32 * m * m, where 32 comes from the nn.Conv2d(3, **32**, kernel_size=7, stride=2), but then m would equals 13. However, where is the 13 comes from?

simple_model = nn.Sequential(
                nn.Conv2d(3, 32, kernel_size=7, stride=2),
                nn.ReLU(inplace=True),
                Flatten(), # see above for explanation
                nn.Linear(5408, 10), # affine layer
              )

Case 2 How we get the number 4*4 in fc = nn.Linear(64*4*4, 10), the same problem in case 1. I don't know where the number 4 comes from...

# (conv -> batchnorm -> relu -> maxpool) * 3 -> fc

layer1 = nn.Sequential(
    nn.Conv2d(3, 16, kernel_size = 5, padding = 2),
    nn.BatchNorm2d(16),
    nn.ReLU(),
    nn.MaxPool2d(2)
)
layer2 = nn.Sequential(
    nn.Conv2d(16, 32, kernel_size = 3, padding = 1),
    nn.BatchNorm2d(32),
    nn.ReLU(),
    nn.MaxPool2d(2)
)
layer3 = nn.Sequential(
    nn.Conv2d(32, 64, kernel_size = 3, padding = 1),
    nn.BatchNorm2d(64),
    nn.ReLU(),
    nn.MaxPool2d(2)
)
fc = nn.Linear(64*4*4, 10)

Output:
Here is a good primer (specifically the summary) on calculating these sorts of things:
 http://cs231n.github.io/convolutional-networks/

Where

W = Input width
Fw = Receptive field (kernel) width
P = Padding
Sw = Stride along width

You haven't mentioned your input width/height, but I'm assuming they're 28x28 MNIST images.

In that case we have:

W = 28
Fw = 7
P = 2
Sw = 2

Plugging those numbers into the above equation will give you 13.5 which is awkward because it's not an integer. In the case of PyTorch it seems to round it down to 13. (It's actually proven kind of hard to find any documentation of this fact other than this forum post)

Edit: The actual implementation for cuDNN is here: https://github.com/pytorch/pytorch/blob/fdab1cf0d485820907d7541266d69b70e1d3d16b/aten/src/ATen/native/cudnn/Conv.cpp#L157-L158

For your second case, it seems like your inputs are not 28x28 and must be 32x32. The convolutions don't shrink the height and width (you can plug the numbers in yourself and check). However the MaxPool2d(2) layer shrinks the height and width  by half after each convolution. So you go from:

32x32->16x16->8x8-> 4x4


Instruction: Forward method calling in Linear class in Pytorch

Input:
I am trying to use Linear class of Pytorch module.In this automatically forward method is called once I create an instance of the class.Can anyone explain me from basics (If I can get code of Linear class of pytorch that would be great)

from torch.nn mport Linear
model=Linear(in_features=1,out_features=1)
x=torch.tensor([1])
y=model(x)

I am not able to understand how model(x) displays the result.As per my understanding model.forward() should display the actual result
It would be great if someone explain me from basics (Would be great if someone can share the link of code of Linear class in pytorch)

Output:
In the source, you'll see that the __call__ method of any Module in turn calls the forward method, so calling model(x) in your case is kind of like calling model.forward(x), but not exactly, due to the reasons explained here.

If you're not familiar with the __call__ method, just know roughly that it makes an object of a class callable.

This same question is answered in this Pytorch forum post.


Instruction: Create all binary valued vectors of specific length

Input: How do I efficiently create a matrix of shape (2**N, N) with all binary vectors of length N as its rows? I call a vector binary iff each of its components equals either 0 or 1. I want the matrix to have dtype=torch.float.

Output:
Seb's answer but modified for pytorch:

def gen_all_binary_vectors(length: int) -> torch.Tensor:
    return ((torch.arange(2**length).unsqueeze(1) >> torch.arange(length-1, -1, -1)) & 1).float()


Instruction: Best way to create a tic-tac-toe model with pytorch

Input:
I want to create a model that can predict a 9x9 tic-tac-toe winner based on player moves.
Here is an example from my dataset:

..................................x.............................................. 14 L
..............o...................x.................x............................ 67 L
..............o...............x...x.................x..............o............. 2 L

There are 81 fields that can be X or o. Number on the left is the next step of the opponent(opponent is always 0). And letter represents the outcome of the game. I decided to replace all "."  with 0, x with 1 and o with 2. L or W with one hot encoding. I compress my positions with future step and feed it into the model. That's where i have trouble. My train_x dimension is (249561, 80, 1). My sample train is

tensor([0, 0, 0, 0, 1, 2, 1, 2, 0, 0, 2, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 2, 2, 2,
        0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 0, 2, 0, 1, 0, 0, 0, 2, 0, 1, 0, 1, 1,
        1, 0, 2, 0, 0, 1, 2, 1, 0, 0, 1, 2, 2, 1, 1, 0, 0, 0, 0, 0, 2, 1, 0, 2,
        0, 0, 1, 0, 0, 0, 2, 0], [67])

I tried this

self.fc = nn.Sequential(
        nn.Linear(80, 4096),
        nn.ReLU(),
        nn.Dropout(p=0.5),
        nn.Linear(4096, 2048),
        nn.ReLU(),
        nn.Dropout(p=0.5),
        nn.Linear(2048, 1),
    )

def forward(self, x):
    logit = self.fc(x)
    return logit

When i am going through training loop I get an error RuntimeError: Expected object of scalar type Float but got scalar type Long for argument #2 'mat1' in call to _th_addmm
I have two questions. Am i doing the data processing right and what model should I use?

Output:
To solve you RuntimeError, which is very descriptive, you must simply cast your tensor from Long to Float:

input_sample = input_sample.float()

Or, when building the samples, change the replacement from [0, 1, 2] (Long) to [0., 1., 2.] (Float).


Instruction: What does local rank mean in distributed deep learning?

Input:
https://github.com/huggingface/transformers/blob/master/examples/run_glue.py

I want to adapt this script to do text classification on my data. The computer for this task is one single machine with two graphic cards. So this involves kind of "distributed" training with the term local_rank in the script above, especially when local_rank equals 0 or -1 like in line 83.

After reading some materials from distributed computation I guess that local_rank is like an ID for a machine. And 0 may mean this machine is the "main" or "head" in the computation. But what is -1?

Output:
  Q: But what is -1?

Usually, this is used to disable the distributed setting. Indeed, as you can see here:

train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)

and here:

if args.local_rank != -1:
    model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
                                                      output_device=args.local_rank,
                                                      find_unused_parameters=True)

setting local_rank to -1 has this effect.


Instruction: Can't we run an onnx model imported to pytorch?

Input:
I have been trying to import a model from onnx format to work with pytorch. I am finding it difficult to get an example for the same. As most of the resources in Internet talks about exporting a pytorch model to onnx.

I found that torch.onnx() can only export the model and the import method hasn't been implemented yet. A direct installation of onnx library, helps me to do onnx.load("model_name.onnx"). How do I use this model with pytorch? I am not able to move the model to GPU with model.to(device="GPU")

Output:
PyTorch doesn't currently support importing onnx models. As of writing this answer it's an open feature request.
While not guaranteed to work, a potential solution is to use a tool developed by Microsoft called MMdnn (no it's not windows only!) which supports conversion to and from various frameworks. Unfortunately onnx can only be a target of a conversion, and not a source. That said you may be able to import your model to another framework, then use MMdnn to convert from that framework to pytorch. Obviously this isn't ideal and the potential for success will depend on how other frameworks use onnx which may not be amenable to the way MMdnn works.
Update August 2022
Unfortunately it appears the feature request was rejected and Mmdnn has been abandoned. There are some more recent 3rd party tools that provide some ability to import onnx into pytorch like onnx2pytorch and onnx-pytorch. Neither of these tools appear to be actively developed, though pytorch and onnx are relatively stable  at this point so hopefully these tools remain relevant in the future (official support would be better IMO). Note that both of these tools have unaddressed issues, so it may be necessary to try both of them if one doesn't work for you.
Update September 2022
Based on the comment from @DanNissenbaum there is a newer 3rd party tool onnx2torch that is being actively developed and maintained.


Instruction: How does the __getitem__'s idx work within PyTorch's DataLoader?

Input:
I'm currently trying to use PyTorch's DataLoader to process data to feed into my deep learning model, but am facing some difficulty.
The data that I need is of shape (minibatch_size=32, rows=100, columns=41). The __getitem__ code that I have within the custom Dataset class that I wrote looks something like this:
def __getitem__(self, idx):
    x = np.array(self.train.iloc[idx:100, :])
    return x

The reason I wrote it like that is because I want the DataLoader to handle input instances of shape (100, 41) at a time, and we have 32 of these single instances.
However, I noticed that contrary to my initial belief the idx argument the DataLoader passes to the function is not sequential (this is crucial because my data is time series data). For example, printing the values gave me something like this:
idx = 206000
idx = 113814
idx = 80597
idx = 3836
idx = 156187
idx = 54990
idx = 8694
idx = 190555
idx = 84418
idx = 161773
idx = 177725
idx = 178351
idx = 89217
idx = 11048
idx = 135994
idx = 15067

Is this normal behavior? I'm posting this question because the data  batches that are being returned are not what I initially wanted them to be.
The original logic that I used to preprocess the data before using the DataLoader was:

Read data in from either txt or csv file.
Calculate how many batches are in the data and slice the data accordingly. For example, since one input instance is of shape (100, 41) and 32 of these form one minibatch, we usually end up with around 100 or so batches and reshape the data accordingly.
One input is of shape (32, 100, 41).

I'm not sure how else I should be handling the DataLoader hook methods. Any tips or advice are greatly appreciated. Thanks in advance.

Output:
What defines the idx is the sampler or batch_sampler, as you can see here (open-source projects are your friend). In this code (and comment/docstring) you can see the difference between sampler and batch_sampler. If you look here you'll see how the index is chosen:

def __next__(self):
    index = self._next_index()

# and _next_index is implemented on the base class (_BaseDataLoaderIter)
def _next_index(self):
    return next(self._sampler_iter)

# self._sampler_iter is defined in the __init__ like this:
self._sampler_iter = iter(self._index_sampler)

# and self._index_sampler is a property implemented like this (modified to one-liner for simplicity):
self._index_sampler = self.batch_sampler if self._auto_collation else self.sampler

Pay attention that this is the _SingleProcessDataLoaderIter implementation; you can find the _MultiProcessingDataLoaderIter here (ofc, which one is used depends on the num_workers value, as you can see here). Going back to the samplers, assuming your Dataset is not _DatasetKind.Iterable and that you are not providing a custom sampler, it means you are either using (dataloader.py#L212-L215):

if shuffle:
    sampler = RandomSampler(dataset)
else:
    sampler = SequentialSampler(dataset)

if batch_size is not None and batch_sampler is None:
    # auto_collation without custom batch_sampler
    batch_sampler = BatchSampler(sampler, batch_size, drop_last)

Let's take a look at how the default BatchSampler builds a batch:

def __iter__(self):
    batch = []
    for idx in self.sampler:
        batch.append(idx)
        if len(batch) == self.batch_size:
            yield batch
            batch = []
    if len(batch) > 0 and not self.drop_last:
        yield batch

Very simple: it gets indices from the sampler until the desired batch_size is reached.

Now the question "How does the __getitem__'s idx work within PyTorch's DataLoader?" can be answered by seeing how each default sampler works.

SequentialSampler (this is the full implementation -- very simple, isn't it?):

class SequentialSampler(Sampler):
    def __init__(self, data_source):
        self.data_source = data_source

    def __iter__(self):
        return iter(range(len(self.data_source)))

    def __len__(self):
        return len(self.data_source)

RandomSampler (let's see only the __iter__ implementation):

def __iter__(self):
    n = len(self.data_source)
    if self.replacement:
        return iter(torch.randint(high=n, size=(self.num_samples,), dtype=torch.int64).tolist())
    return iter(torch.randperm(n).tolist())

Therefore, as you did not provide any code, we can only assume:

You are using shuffle=True in your DataLoader or
You are using a custom sampler or
Your Dataset is _DatasetKind.Iterable


Instruction: 'iterator' object has no attribute 'next' in python 3.7

Input:
I'm trying to iterate through my data set and get the first element
    transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5),(0.5)),])
    trainloader = datasets.MNIST('~/.pytorch/MNIST_data' , download=True,train=True , transform=transform)
    ds = iter(trainloader)
    img, labels = ds.next()

but it returns this error
    AttributeError: 'iterator' object has no attribute 'next'

I also tried this
    img , labels = next(ds)

returned this error
    StopIteration:

Did I miss something ?

Output:
If you follow the tutorial on https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                          shuffle=True, num_workers=2)

dataiter = iter(trainloader)
images, labels = dataiter.next()

You are missing the DataLoader() function on your dataset


Instruction: How to define a loss function in pytorch with dependency to partial derivatives of the model w.r.t input?

Input:
After reading about how to solve an ODE with neural networks following the paper Neural Ordinary Differential Equations and the blog that uses the library JAX I tried to do the same thing with "plain" Pytorch but found a point rather "obscure": How to properly use the partial derivative of a function (in this case the model) w.r.t one of the input parameters.

To resume the problem at hand as shown in 2 it is intended to solve the ODE y' = -2*x*y with the condition y(x=0) = 1 in the domain -2 <= x <= 2. Instead of using finite differences the solution is replaced by a NN as y(x) = NN(x) with a single layer with 10 nodes.

I managed to (more or less) replicate the blog with the following code

import torch
import torch.nn as nn
from torch import optim
import matplotlib.pyplot as plt
import numpy as np

# Define the NN model to solve the problem
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,10)
        self.lin2 = nn.Linear(10,1)

    def forward(self, x):
        x = torch.sigmoid(self.lin1(x))
        x = torch.sigmoid(self.lin2(x))
        return x

model = Model()

# Define loss_function from the Ordinary differential equation to solve
def ODE(x,y):
    dydx, = torch.autograd.grad(y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True)

    eq = dydx + 2.* x * y # y' = - 2x*y
    ic = model(torch.tensor([0.])) - 1.    # y(x=0) = 1
    return torch.mean(eq**2) + ic**2

loss_func = ODE

# Define the optimization
# opt = optim.SGD(model.parameters(), lr=0.1, momentum=0.99,nesterov=True) # Equivalent to blog
opt = optim.Adam(model.parameters(),lr=0.1,amsgrad=True) # Got faster convergence with Adam using amsgrad

# Define reference grid
x_data = torch.linspace(-2.0,2.0,401,requires_grad=True)
x_data = x_data.view(401,1) # reshaping the tensor

# Iterative learning
epochs = 1000
for epoch in range(epochs):
    opt.zero_grad()
    y_trial = model(x_data)
    loss = loss_func(x_data, y_trial)

    loss.backward()
    opt.step()

    if epoch % 100 == 0:
        print('epoch {}, loss {}'.format(epoch, loss.item()))

# Plot Results
plt.plot(x_data.data.numpy(), np.exp(-x_data.data.numpy()**2), label='exact')
plt.plot(x_data.data.numpy(), y_data.data.numpy(), label='approx')
plt.legend()
plt.show()

From here I manage to get the results as shown in the fig.
enter image description here

The problems is that at the definition of the ODE functional, instead of passing (x,y) I would rather prefer to pass something like (x,fun) (where fun is my model) such that the partial derivative and specific evaluations of the model can be done with a call . So, something like

def ODE(x,fun):
    dydx, = "grad of fun w.r.t x as a function"

    eq = dydx(x) + 2.* x * fun(x)        # y' = - 2x*y
    ic = fun( torch.tensor([0.]) ) - 1.  # y(x=0) = 1
    return torch.mean(eq**2) + ic**2

Any ideas? Thanks in advance

EDIT:

After some trials I found a way to pass the model as an input but found another strange behavior... The new problem is to solve the ODE y'' = -2 with the BC y(x=-2) = -1 and y(x=2) = 1, for which the analytical solution is y(x) = -x^2+x/2+4

Let's modify a bit the previous code as:

import torch
import torch.nn as nn
from torch import optim
import matplotlib.pyplot as plt
import numpy as np

# Define the NN model to solve the equation
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,10)
        self.lin2 = nn.Linear(10,1)

    def forward(self, x):
        y = torch.sigmoid(self.lin1(x))
        z = torch.sigmoid(self.lin2(y))
        return z

model = Model()

# Define loss_function from the Ordinary differential equation to solve
def ODE(x,fun):
    y = fun(x)

    dydx = torch.autograd.grad(y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True)[0]

    d2ydx2 = torch.autograd.grad(dydx, x,
    grad_outputs=dydx.data.new(dydx.shape).fill_(1),
    create_graph=True, retain_graph=True)[0]

    eq  = d2ydx2 + torch.tensor([ 2.])             # y'' = - 2
    bc1 =  fun(torch.tensor([-2.])) - torch.tensor([-1.]) # y(x=-2) = -1
    bc2 =  fun(torch.tensor([ 2.])) - torch.tensor([ 1.]) # y(x= 2) =  1
    return torch.mean(eq**2) + bc1**2 + bc2**2

loss_func = ODE

So, here I passed the model as argument and managed to derive twice... so far so good. BUT, using the sigmoid function for this case is not only not necessary but also gives a result that is far from the analytical one.

If I change the NN for:

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.lin1 = nn.Linear(1,1)
        self.lin2 = nn.Linear(1,1)

    def forward(self, x):
        y = self.lin1(x)
        z = self.lin2(y)
        return z

In which case I would expect to optimize a double pass through two linear functions that would retrieve a 2nd order function ... I get the error:

  RuntimeError: One of the differentiated Tensors appears to not have been used in the graph. Set allow_unused=True if this is the desired behavior.

Adding the option to the definition of dydx doesn't solve the problem, and adding it to d2ydx2 gives a NoneType definition.

Is there something wrong with the layers as they are?

Output:
Quick Solution:

add allow_unused=True to .grad functions. So, change

dydx = torch.autograd.grad(
    y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True)[0]

d2ydx2 = torch.autograd.grad(dydx, x, grad_outputs=dydx.data.new(
    dydx.shape).fill_(1), create_graph=True, retain_graph=True)[0]

To

dydx = torch.autograd.grad(
    y, x,
    grad_outputs=y.data.new(y.shape).fill_(1),
    create_graph=True, retain_graph=True, allow_unused=True)[0]

d2ydx2 = torch.autograd.grad(dydx, x, grad_outputs=dydx.data.new(
    dydx.shape).fill_(1), create_graph=True, retain_graph=True, allow_unused=True)[0]

More explanation:

See what allow_unused do:

allow_unused (bool, optional): If ``False``, specifying inputs that were not
        used when computing outputs (and therefore their grad is always zero)
        is an error. Defaults to ``False``.

So, if you try to differentiate w.r.t to a variable that is not in being used to compute the value, it will give an error. Also, note that error only occurs when you use linear layers.

This is because when you use linear layers, you have y=W1*W2*x + b = Wx+b and dy/dx is not a function of x, it is simply W. So when you try to differentiate dy/dx w.r.t x it throws an error. This error goes away as soon as you use sigmoid because then dy/dx will be a function of x. To avoid the error, either make sure dy/dx is a function of x or use allow_unused=True


Instruction: Adjust input shape for my pytorch problem

Input:
I am trying to create a neural net to feed in an input of shape 249561, 80, 1 and y labels are (249561, 2).

def __init__(self):
    super(Net1, self).__init__()
    self.conv1 = nn.Conv1d(80, 16, kernel_size=1)

    self.conv2_drop = nn.Dropout()
    self.fc1 = nn.Linear(1,256)
    self.fc2 = nn.Linear(256, 64)
    self.fc3 = nn.Linear(64,32)
    self.fc4 = nn.Linear(32,2)

def forward(self, x):
    print(type(x))
    x = F.relu(F.max_pool1d(self.conv1(x), 1))
    print(x.shape)
    x.reshape(-1)
    e1 = F.relu(self.fc1(x))
    x = F.dropout(e1, training=self.training)
    x = F.relu(self.fc2(x))
    x = F.dropout(x, training=self.training)
    x = F.relu(self.fc3(x))
    x = F.dropout(x, training=self.training)
    x = self.fc4(x)

    return x

My training loop looks something like this

losses = [];
batch_size = 16
for epoch in range(10):
      permutation = torch.randperm(x2.size()[0])
      for i in range(0,len(x2), batch_size):
        indices = permutation[i:i+batch_size]
        batch_x, batch_y = x2[indices], onehot_encoded[indices]
        #images = Variable(images.float())
        #labels = Variable(labels)

        # Forward + Backward + Optimize
        optimizer.zero_grad()
        outputs = model(batch_x)
        loss = criterion(outputs, batch_y)
        loss.backward()
        optimizer.step()

I have a batch of 16 and I feed in a tensor with a shape [16, 80, 1].I get the following error. RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss2d_forward. I suspect that it is the problem with an output layer but it returns a tensor of shape 2 which is the same as my labels. The output x size torch.Size([16, 16, 2])

Output:
Instead of changing the input, why don't you use nn.Conv1d (replace both nn.Conv2d -- you'll need to change the dropout as well)?

If you really wanna change the input, you can add:

batch_x = batch_x[..., None]

After

batch_x, batch_y = x2[indices], onehot_encoded[indices]


Instruction: Pytorch + Residual Network throws unexpected: TypeError: 'NoneType' object is not callable

Input:
I am diving into the world of Deep Learning and as Python is my most prominent programming language I decided to start with the Pytorch framework. Following a tutorial I implemented a ResNet of 50 layers. Afterwards I tried to make an ResNet-18 with simpler blocks which looked like this:

class baseBlock(torch.nn.Module):
  expansion = 1
  def __init__(self,input_planes,planes,stride=1,dim_change=None):
    super(baseBlock,self).__init__()
    #Declare convolutional layers with batch norms
    self.conv1 = torch.nn.Conv2d(input_planes, planes, stride=stride, kernel_size=3, padding=1)
    self.bn1   = torch.nn.BatchNorm2d(planes)
    self.conv2 = torch.nn.Conv2d(planes, planes, stride=1, kernel_size=3, padding=1)
    self.bn2   = torch.nn.BatchNorm2d(planes)
    self.dim_change = dim_change

  def forward(self,x):
    #Save the residue
    res = x
    output = F.relu(self.bn1(self.conv1(x)))
    output = self.bn2(self.conv2(output))

    if self.dim_change is not None:
      res = self.dim_change(res)

    output += res
    output = F.relu(output)

    return output

I divided the code into the blocks and the actual Neural Network to make it more understandable. The rest of the code for the NN is:

class ResNet(torch.nn.Module):
  def __init__(self,block,num_layers,classes=10):
    super(ResNet,self).__init__()
    #according to research paper
    self.input_planes = 64
    self.conv1  = torch.nn.Conv2d(3,64,kernel_size =3,stride=1,padding=1)
    self.bn1    = torch.nn.BatchNorm2d(64)
    self.layer1 = self._layer(block, 64, num_layers[0], stride=1)
    self.layer2 = self._layer(block, 128, num_layers[1], stride=2)
    self.layer3 = self._layer(block, 256, num_layers[2], stride=2)
    self.layer4 = self._layer(block, 512, num_layers[3], stride=2)
    self.avaragePool = torch.nn.AvgPool2d(kernel_size=4,stride=1)
    self.fc = torch.nn.Linear(512*block.expansion, classes)

  def _layer(self,block,planes,num_layers,stride=1):
    dim_change = None
    if stride != 1 or planes != self.input_planes*block.expansion:
      dim_change = torch.nn.Sequential(torch.nn.Conv2d(self.input_planes, planes*block.expansion, kernel_size = 1, stride=stride),
                                                        torch.nn.BatchNorm2d(planes*block.expansion))
      netLayers = []
      netLayers.append(block(self.input_planes,planes,stride=stride, dim_change=dim_change))

      self.input_planes = planes*block.expansion

      for i in range(1,num_layers):
        netLayers.append(block(self.input_planes,planes))
        self.input_planes = planes * block.expansion

      return torch.nn.Sequential(*netLayers)

  def forward(self,x):
    x = F.relu(self.bn1(self.conv1(x)))

    #Problem at this layer
    x = self.layer1(x) <- NoneType object is not callable
    x = self.layer2(x)
    x = self.layer3(x)
    x = self.layer4(x)

    x = F.avg_pool2d(x,4)
    x = x.view(x.size(0),-1)
    x = self.fc(x)

    return x

The problem arises on the last forward function in the layer1 that is marked on the code. If I comment it the network keeps on computing (obviously it throws an error later because the dimensions are not right but it still does something). If I change the stride on this first layer the problem also resolves but still throws an error later for the same dimensional reasons. I don't seem to figure out what the problem is. I use a standard CIFAR10 dataset to train it and the training code is as standard as it gets.

If I can provided any additional information let me know. Thank you

Output:
This is a simple typo. Your return is inside the if, and the only layer with stride==1 is the layer1:

def _layer(self,block,planes,num_layers,stride=1):
    dim_change = None
    if stride != 1 or planes != self.input_planes*block.expansion:
        # [...]
        return torch.nn.Sequential(*netLayers)

Therefore, for the layer1, there is no return, hence None.


Instruction: matrix multiplication using star operator

Input:
I am curious about what the difference is between calling torch.mm(A, B) and A*B?

Looks like torch.mm gives us the desirable result, but A*B sometimes doesn't work.

It will be better if any documentation is provided.

Thank you!

Output: torch.mm(A,B) is a regular matrix multiplication and A*B is element-wise multiplication. You can read it on this discussion. For matrix multiplication you can use @ if I am not mistaken as well.


Instruction: How do loss functions know for which model to compute gradients in PyTorch?

Input:
I am unsure how PyTorch manges to link the loss function to the model I want it to be computed for. There is never an explicit reference between the loss and the model, such as the one between the model's parameters and the optimizer.

Say for example I want to train 2 networks on the same dataset, so I want to utilize a single pass through the dataset. How would PyTorch link the appropriate loss functions to the appropriate models. Here's code for reference:

import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
import shap

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                                transforms.Normalize((0.5,), (0.5,)),
                              ])
# Download and load the training data
trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)

model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 64),
                      nn.ReLU(),
                      nn.Linear(64, 10),
                      nn.LogSoftmax(dim=1))

model2 = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 10),
                      nn.LogSoftmax(dim=1))

# Define the loss
criterion = nn.NLLLoss()
criterion2 = nn.NLLLoss()

optimizer = optim.SGD(model.parameters(), lr=0.003)
optimizer2 = optim.SGD(model2.parameters(), lr=0.003)

epochs = 5
for e in range(epochs):
    running_loss = 0
    running_loss_2 = 0
    for images, labels in trainloader:
        # Flatten MNIST images into a 784 long vector
        images = images.view(images.shape[0], -1) # batch_size x total_pixels

        # Training pass
        optimizer.zero_grad()
        optimizer2.zero_grad()

        output = model(images)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()

        output2 = model2(images)
        loss2 = criterion2(output2, labels)
        loss2.backward()
        optimizer2.step()

        running_loss += loss.item()
        running_loss_2 += loss2.item()

    print(f"Training loss 1: {running_loss/len(trainloader)}")
    print(f"Training loss 2: {running_loss_2/len(trainloader)}")
    print()

So once again, how does pytorch know how to compute the appropriate gradients for the appropriate models when loss.backward() and loss2.backward() are called?

Output:
Whenever you perform forward operations using one of your model parameters (or any torch.tensor that has attribute requires_grad==True), pytorch builds a computational graph. When you operate on descendents in this graph, the graph is extended. In your case, you have a nn.module called model which will have some trainable model.parameters(), so pytorch will build a graph from your model.parameters() all the way to the loss as you perform the forward operations. The graph is then traversed in reverse during the backward pass to propagate the gradients back to the parameters. For  loss in your code above the graph is something like

model.parameters() --> [intermediate variables in model] -->  output --> loss
                                  ^                                        ^
                                  |                                        |
                               images                                     labels

When you call loss.backward() pytorch traverses this graph in reverse to reach all trainable parameters (only the model.parameters() in this case) and updates param.grad for each of them. The optimizer then relies on this information gathered during the backward pass to update the parameter.
For loss2 the story is similar.

The official pytorch tutorials are a good resource for more in-depth information on this.


Instruction: Is there an equivalent PyTorch function for `tf.nn.space_to_depth`

Input: As the title says, is there an equivalent PyTorch function for tf.nn.space_to_depth?

Output:
While torch.nn.functional.pixel_shuffle does exactly what tf.nn.depth_to_space does, PyTorch doesn't have any function to do the inverse operation similar to tf.nn.space_to_depth.

That being said, it is easy to implement space_to_depth using torch.nn.functional.unfold.

def space_to_depth(x, block_size):
    n, c, h, w = x.size()
    unfolded_x = torch.nn.functional.unfold(x, block_size, stride=block_size)
    return unfolded_x.view(n, c * block_size ** 2, h // block_size, w // block_size)


Instruction: How to install an older version of gcc on Fedora

Input:
I tried to install CUDA on Fedora 31 but got stuck at the last step because CUDA officially supports Fedora 29 (gcc 8.2) while the version shipped with Fedora 31 is 9.2, I then installed Pytorch with CUDA support and without much surprise, CUDA support is not present:
>>> import torch
>>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
>>> device
device(type='cpu')

I then tried to search for gcc, g++ and gfortran on pkgs, but got stuck in a dependency hell, something like:
sudo dnf install gcc-8.2.1-2.fc29.x86_64.rpm gcc-gfortran-8.2.1-2.fc29.x86_64.rpm gcc-c++-8.2.1-2.fc29.x86_64.rpm -y

Error:
 Problem 1: conflicting requests
  - nothing provides cpp = 8.2.1-2.fc29 needed by gcc-8.2.1-2.fc29.x86_64
  - nothing provides libgomp = 8.2.1-2.fc29 needed by gcc-8.2.1-2.fc29.x86_64
 Problem 2: conflicting requests
  - nothing provides libgfortran = 8.2.1-2.fc29 needed by gcc-gfortran-8.2.1-2.fc29.x86_64
  - nothing provides libquadmath = 8.2.1-2.fc29 needed by gcc-gfortran-8.2.1-2.fc29.x86_64
  - nothing provides libquadmath-devel = 8.2.1-2.fc29 needed by gcc-gfortran-8.2.1-2.fc29.x86_64
 Problem 3: conflicting requests
  - nothing provides libstdc++ = 8.2.1-2.fc29 needed by gcc-c++-8.2.1-2.fc29.x86_64
  - nothing provides libstdc++-devel = 8.2.1-2.fc29 needed by gcc-c++-8.2.1-2.fc29.x86_64

I found a bunch helpful links on this post but unfortunately the last version is gcc 7.3
My last option is building from source, but apart from that is there any other option? I'm not sure my computer can handle the compilation in a decent amount of time.

Output:
Build gcc-8.2.0
# dnf install gcc-c++ flex bison binutils-devel elfutils-devel elfutils-libelf-devel texinfo doxygen zlib-devel
tar xvf gcc-8.2.0.tar.xz
cd gcc-8.2.0/
tar xvf mpfr-4.0.2.tar.xz && mv -v mpfr-4.0.2 mpfr
tar xvf gmp-6.1.2.tar.xz && mv -v gmp-6.1.2 gmp
tar xvf mpc-1.1.0.tar.gz && mv -v mpc-1.1.0 mpc
cd ../
mkdir build-gcc820
cd build-gcc820/
../gcc-8.2.0/configure --prefix=/usr/local/gcc82 --program-suffix=82 --enable-languages=c,c++,fortran --disable-multilib --disable-libstdcxx-pch --with-system-zlib
make
# make install

Result : gcc82-c++-8.2.0-2.fc29.x86_64.rpm https://drive.google.com/file/d/1qGPvF9jc6CiI1a0-c3I4Zj4vxajEwSzc/view?usp=sharing
Provides /usr/bin/{gcc8, g++8, gfortran8}
Install : # dnf install Downloads/gcc82-c++-8.2.0-2.fc29.x86_64.rpm


Instruction: Loading and correctly displaying an image dataset using pytorch Dataloader

Input:
I'm trying to load a custom dataset for training a neural network, but before I load them in, I would like to verify that they've been loaded correctly. So far it looks like they are not being loaded correctly, but I can't figure out what gives the images the format that they get.

This is the code that I'm loading the images with, and then displaying them.

f, axarr = plt.subplots(2,2, figsize=(20,20))

def load_dataset():
    data_path = 'processedData/HE/train/'
    train_dataset = torchvision.datasets.ImageFolder(
        root=data_path,
        transform=torchvision.transforms.ToTensor()
    )
    train_loader = DataLoader(
        train_dataset, batch_size=64
    )
    return train_loader
x_train = load_dataset()

datathing = next(iter(x_train))

for i, ax in enumerate(axarr.flat):
    ax.imshow(datathing[0][i].view(128,128,3))
    ax.axis('off')
plt.show()

When running this with the images, the output looks like this.

It is suppose to look something like these images

I have been trying with different image datasets, but all the sets return the same format, so my question is:

How to load images and then displaying them in their real format using pytorch's dataloader?

Output:
The .view(128, 128, 3) is messing up with the images.
As you can read in the documentation of the transformation .ToTensor(...):

[...]
Converts a PIL Image or numpy.ndarray (H x W x C) in the range [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0]
[...]

That is, the channel dimension is moved from the last to the first dimension. You can see that in the source code:
# ...
img = torch.from_numpy(pic.transpose((2, 0, 1)))
# ...

Therefore, you cannot simply call .view(...); you have to transpose it back. In PyTorch, you can use the .permute(...) function for that. Something like this:
ax.imshow(datathing[0][i].permute(1, 2, 0))


Instruction: PyTorch - How to Save Custom Dataset to disk to use with torchvision.datasets?

Input:
I'd like to create a custom PyTorch dataset of ZCA-whitened CIFAR-10 that I can subsequently load using torchvision's function torchvision.datasets.CIFAR10(). So far, I can successfully whiten the data (see code below), but I don't know how to save the data to disk in a manner that allows it to be loaded using torchvision.datasets.CIFAR10(). How do I do this?

Code to ZCA-whiten CIFAR 10:

    trainset = torchvision.datasets.CIFAR10(
        root='./datasets',
        train=True,
        download=False)
    train_data = trainset.data.reshape(-1, 32*32*3)
    zca_matrix = zca_whitening_matrix(train_data.T)
    whitened_training_data = np.matmul(zca_matrix, train_data.T).T
    whitened_training_data = whitened_training_data.reshape((-1, 32, 32, 3))

    # whiten CIFAR-10 testing data
    testset = torchvision.datasets.CIFAR10(
        root='./datasets',
        train=False,
        download=False)
    testdata = testset.data.reshape(-1, 32*32*3)
    whitened_test_data = np.matmul(zca_matrix, testdata.T).T
    whitened_test_data = whitened_test_data.reshape((-1, 32, 32, 3))

Is the best way to really just save the numpy arrays, as shown here?

PyTorch: How to use DataLoaders for custom Datasets

Output:
This is not possible by default for any of the datasets.

The recommended way is to subclass and create functionality yourself, or to manky patch the class with a new method.


Instruction: Is conv2d using random filters? Is that good for the NN?

Input:
If I am using nn.Conv2d(in_dim, out_dim, kernel_size, bias=False) command in PyTorch, then the function creates as many filters as specified by out_dim. Are these filters random? Is it not be better to customize these filters? Does someone have a method for this?

Or is it better to use the random filter generated by this function?

Done in PyTorch, python.

Thanks

Output:
  Are these filters random? Is it not be better to customize these filters? Does someone have a method for this?

Yes, they are pretty random, made via the initialization function used for the Conv2d.

This may change but by default, nn.Conv2d uses init.kaiming_uniform_. You can tweak it as:

conv_layer = nn.Conv2d(in_dim, out_dim, kernel_size, bias=False)
torch.nn.init.kaiming_normal_(conv_layer.weight)

The concept is to learn the weights while training, so these random values at first will alter and become learned tensor values.


Instruction: Something wrong happens when shuffle a tensor

Input:
I am trying to shuffle a tensor, but I find there is something wrong when tryting to using random.shuffle(),here is my code:

import numpy as np
import torch
import random

if __name__ == '__main__':

    label_0, label_1 = np.zeros([1, 1000])[0], np.ones([1, 1000])[0]
    test_label = np.hstack((label_0, label_1))

    num_0, num_1 = 0, 0

    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('Num1:  ', num_1)
    print('Num0:  ', num_0)

    random.shuffle(test_label)

    num_0, num_1 = 0, 0
    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('After Shuffle Num1:  ', num_1)
    print('After Shuffle Num0:  ', num_0)

If test_label is not a tensor, it works perfectly:

Num1:   1000
Num0:   1000
After Shuffle Num1:   1000
After Shuffle Num0:   1000

However when I try to transform the test_label from ndarray to tensor, something wrong would happend:

import numpy as np
import torch
import random

if __name__ == '__main__':

    label_0, label_1 = np.zeros([1, 1000])[0], np.ones([1, 1000])[0]
    test_label = np.hstack((label_0, label_1))

    # now transform to tensor
    test_label = torch.tensor(test_label)

    num_0, num_1 = 0, 0

    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('Num1:  ', num_1)
    print('Num0:  ', num_0)

    random.shuffle(test_label)

    num_0, num_1 = 0, 0
    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('After Shuffle Num1:  ', num_1)
    print('After Shuffle Num0:  ', num_0)

And the result seems to be kind wrong:

Num1:   1000
Num0:   1000
After Shuffle Num1:   306
After Shuffle Num0:   1694

Anybody can tell me why will make such problem?

Output:
You can use torch.randperm:

import numpy as np
import torch
import random

if __name__ == '__main__':

    label_0, label_1 = np.zeros([1, 1000])[0], np.ones([1, 1000])[0]
    test_label = np.hstack((label_0, label_1))

    # now transform to tensor
    test_label = torch.tensor(test_label)

    num_0, num_1 = 0, 0

    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('Num1:  ', num_1)
    print('Num0:  ', num_0)

    p = torch.randperm(test_label.shape[0])
    test_label = test_label[p]

    num_0, num_1 = 0, 0
    for i in range(len(test_label)):
        if test_label[i] == 0:
            num_0 += 1
        elif test_label[i] == 1:
            num_1 += 1

    print('After Shuffle Num1:  ', num_1)
    print('After Shuffle Num0:  ', num_0)


Instruction: How to read images from pandas column in a pytorch class

Input:
I need to define a pytorch dataset class that reads the absolute path from a column to pull the image. When I try this with multiple packages, I get errors.  Below are the errors for each package I'm aware of:

Pathlib.Path: TypeError: expected str, bytes or os.PathLike object, not method

glob.glob: TypeError: expected str, bytes or os.PathLike object, not method

os.path: TypeError: 'module' object is not callable

Below is my pytorch custom dataset.

class image_Dataset(Dataset):
'''
image data class

'''
def __init__(self, data, transform = None):
    '''
    Args:
    ------------------------------------------------------------
        data = dataframe
        image = column in dataframe with absolute path to the image
        label = column in dataframe that is the target classification variable
        policy = ID variable

    '''
    self.image_frame = data
    self.transform = transform

def __len__(self):
    return len(self.image_frame)

def __getitem__(self, idx):
    if torch.is_tensor(idx):
        idx = idx.tolist()

    label = self.image_frame.iloc[idx, 16]
    pic = Path(self.image_frame.iloc[idx,19])
    img = Image.open(pic)
    policy = self.image_frame.iloc[idx, 1]
    sample = {'image': img, 'policy': policy, 'label':label}

    if self.transform:
        image = self.transform(image)

    return image, label, policy

path column in my dataset values looks like the following:

D:/Models/Photos/Train/train_img.jpg
Full trace back below:

IndexError                                Traceback (most recent call last)
<ipython-input-261-38143c041795> in <module>
      7
      8 for i in range(len(roof_data_test)):
----> 9     sample = roof_data_test[i]
     10
     11     print(i, sample[0].shape, sample[1], sample[2])

<ipython-input-260-1bd25b938fea> in __getitem__(self, idx)
     25
     26         label = self.image_frame.iloc[idx, 16]
---> 27         pic = Path(self.image_frame.iloc[idx,19])
     28         img = Image.open(pic)
     29         policy = self.image_frame.iloc[idx, 1]

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexing.py in __getitem__(self, key)
   1492             except (KeyError, IndexError, AttributeError):
   1493                 pass
-> 1494             return self._getitem_tuple(key)
   1495         else:
   1496             # we by definition only have the 0th axis

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexing.py in _getitem_tuple(self, tup)
   2141     def _getitem_tuple(self, tup):
   2142
-> 2143         self._has_valid_tuple(tup)
   2144         try:
   2145             return self._getitem_lowerdim(tup)

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexing.py in _has_valid_tuple(self, key)
    221                 raise IndexingError('Too many indexers')
    222             try:
--> 223                 self._validate_key(k, i)
    224             except ValueError:
    225                 raise ValueError("Location based indexing can only have "

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexing.py in _validate_key(self, key, axis)
   2068             return
   2069         elif is_integer(key):
-> 2070             self._validate_integer(key, axis)
   2071         elif isinstance(key, tuple):
   2072             # a tuple should already have been caught by this point

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\indexing.py in _validate_integer(self, key, axis)
   2137         len_axis = len(self.obj._get_axis(axis))
   2138         if key >= len_axis or key < -len_axis:
-> 2139             raise IndexError("single positional indexer is out-of-bounds")
   2140
   2141     def _getitem_tuple(self, tup):

IndexError: single positional indexer is out-of-bounds

Output: single positional indexer is out-of-bounds is telling you that your are trying to access a column that is not there. Make sure that the columns you are trying to access exist


Instruction: Forward vs reverse mode differentiation - Pytorch

Input:
In the first example of Learning PyTorch with Examples, the author demonstrates how to create a neural network with numpy. Their code is pasted below for convenience:

# from: https://pytorch.org/tutorials/beginner/pytorch_with_examples.html
# -*- coding: utf-8 -*-
import numpy as np

# N is batch size; D_in is input dimension;
# H is hidden dimension; D_out is output dimension.
N, D_in, H, D_out = 64, 1000, 100, 10

# Create random input and output data
x = np.random.randn(N, D_in)
y = np.random.randn(N, D_out)

# Randomly initialize weights
w1 = np.random.randn(D_in, H)
w2 = np.random.randn(H, D_out)

learning_rate = 1e-6
for t in range(500):
    # Forward pass: compute predicted y
    h = x.dot(w1)
    h_relu = np.maximum(h, 0)
    y_pred = h_relu.dot(w2)

    # Compute and print loss
    loss = np.square(y_pred - y).sum()
    print(t, loss)

    # Backprop to compute gradients of w1 and w2 with respect to loss
    grad_y_pred = 2.0 * (y_pred - y)
    grad_w2 = h_relu.T.dot(grad_y_pred)
    grad_h_relu = grad_y_pred.dot(w2.T)
    grad_h = grad_h_relu.copy()
    grad_h[h < 0] = 0
    grad_w1 = x.T.dot(grad_h)

    # Update weights
    w1 -= learning_rate * grad_w1
    w2 -= learning_rate * grad_w2

What is confusing to me is why gradients of w1 and w2 are computed with respect to loss (2nd to last code block).

Normally the opposite computation happens: the gradients of loss is computed with respect to the weights, as quoted here:

"When training neural networks, we think of the cost (a value describing how bad a neural network performs) as a function of the parameters (numbers describing how the network behaves). We want to calculate the derivatives of the cost with respect to all the parameters, for use in gradient descent. Now, there’s often millions, or even tens of millions of parameters in a neural network. So, reverse-mode differentiation, called backpropagation in the context of neural networks, gives us a massive speed up!" (Colah's blog).

So my question is: why is the derivation computation in the example above in reverse order as compared to normal back propagation computations?

Output:
Seems to be a typo in the comment. They are actually computing gradient of loss w.r.t. w2 and w1.

Let's quickly derive the gradient of loss w.r.t. w2 just to be sure. By inspection of your code we have

Using the chain rule from calculus

.

Each term can be represented using the basic rules of matrix calculus. These turn out to be

and

.

Plugging these terms back into the initial equation we get

.

Which perfectly matches the expressions described by

grad_y_pred = 2.0 * (y_pred - y)       # gradient of loss w.r.t. y_pred
grad_w2 = h_relu.T.dot(grad_y_pred)    # gradient of loss w.r.t. w2

in the back-propagation code you provided.


Instruction: AttributeError: module 'enum' has no attribute 'IntFlag' - fastai install

Input:
When I try install fastai (pip install fastai) i get following error:

AttributeError: module 'enum' has no attribute 'IntFlag'

Full traceback:

 Installing build dependencies ... error
  ERROR: Complete output from command /usr/bin/python3 /usr/local/lib/python3.6/dist-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-b8vreosn/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel 'numpy==1.13.3; python_version=='"'"'2.7'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.13.3; python_version=='"'"'3.5'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.13.3; python_version=='"'"'3.6'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.14.5; python_version>='"'"'3.7'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'2.7'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'3.5'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'3.6'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version>='"'"'3.7'"'"' and platform_system=='"'"'AIX'"'"'':
  ERROR: Traceback (most recent call last):
    File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
      "__main__", mod_spec)
    File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
      exec(code, run_globals)
    File "/usr/local/lib/python3.6/dist-packages/pip/__main__.py", line 16, in <module>
      from pip._internal import main as _main  # isort:skip # noqa
    File "/usr/local/lib/python3.6/dist-packages/pip/_internal/__init__.py", line 4, in <module>
      import locale
    File "/usr/lib/python3.6/locale.py", line 16, in <module>
      import re
    File "/usr/lib/python3.6/re.py", line 142, in <module>
      class RegexFlag(enum.IntFlag):
  AttributeError: module 'enum' has no attribute 'IntFlag'
  Error in sys.excepthook:
  Traceback (most recent call last):
    File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 53, in apport_excepthook
      if not enabled():
    File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 24, in enabled
      import re
    File "/usr/lib/python3.6/re.py", line 142, in <module>
      class RegexFlag(enum.IntFlag):
  AttributeError: module 'enum' has no attribute 'IntFlag'

  Original exception was:
  Traceback (most recent call last):
    File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
      "__main__", mod_spec)
    File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
      exec(code, run_globals)
    File "/usr/local/lib/python3.6/dist-packages/pip/__main__.py", line 16, in <module>
      from pip._internal import main as _main  # isort:skip # noqa
    File "/usr/local/lib/python3.6/dist-packages/pip/_internal/__init__.py", line 4, in <module>
      import locale
    File "/usr/lib/python3.6/locale.py", line 16, in <module>
      import re
    File "/usr/lib/python3.6/re.py", line 142, in <module>
      class RegexFlag(enum.IntFlag):
  AttributeError: module 'enum' has no attribute 'IntFlag'
  ----------------------------------------
ERROR: Command "/usr/bin/python3 /usr/local/lib/python3.6/dist-packages/pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-b8vreosn/overlay --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools wheel 'numpy==1.13.3; python_version=='"'"'2.7'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.13.3; python_version=='"'"'3.5'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.13.3; python_version=='"'"'3.6'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.14.5; python_version>='"'"'3.7'"'"' and platform_system!='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'2.7'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'3.5'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version=='"'"'3.6'"'"' and platform_system=='"'"'AIX'"'"'' 'numpy==1.16.0; python_version>='"'"'3.7'"'"' and platform_system=='"'"'AIX'"'"''" failed with error code 1 in None

Enum34 cause issues in many packages.I tried to solve this by using:

pip freeze | grep enum34

and

pip uninstall -y enum34

also i tried unset PYTHONPATH and I still get the same error.

pip show enum34 <-- this command give: WARNING: Package(s) not found: enum34

Output:
PROBLEM SOLVED:

as I wrote earlier Enum34 cause issues in many packages.

If you have a similar problem, just use:

pip uninstall -y enum34

My problem was that in my case I should use:

sudo pip uninstall -y enum34


Instruction: Could not find a version that satisfies the requirement torch==1.3.1

Input:
First of all i tried command from their main page, that they gave me:

pip3 install torch==1.3.1+cpu torchvision==0.4.2+cpu -f https://download.pytorch.org/whl/torch_stable.html

Could not find a version that satisfies the requirement torch==1.3.1+cpu (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2)
ERROR: No matching distribution found for torch==1.3.1.+cpu

After this i decided to take available from this list: https://download.pytorch.org/whl/cpu/stable

So at the end i tried something like this

pip3 install torch-1.1.0-cp37-cp37m-win_amd64.whl -f https://download.pytorch.org/whl/torch_stable.html

And now they write that this is not supported wheel on my platform. Wtf?
( I use windows 7, python64, have amd)
( location of python: C:\Python38, location of pip C:\Python38\Scripts )

Output:
There is no any wheels for Python 3.8 at https://download.pytorch.org/whl/torch_stable.html.

  not supported wheel on my platform

This is because the wheel is for Python 3.7.

Advice: downgrade to Python 3.7.


Instruction: Proper way to extract embedding weights for CBOW model?

Input:
I'm currently trying to implement the CBOW model on managed to get the training and testing, but am facing some confusion as to the "proper" way to finally extract the weights from the model to use as our word embeddings.

Model

class CBOW(nn.Module):
    def __init__(self, config, vocab):
        self.config = config # Basic config file to hold arguments.
        self.vocab = vocab
        self.vocab_size = len(self.vocab.token2idx)
        self.window_size = self.config.window_size

        self.embed = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=self.config.embed_dim)
        self.linear = nn.Linear(in_features=self.config.embed_dim, out_features=self.vocab_size)

    def forward(self, x):
        x = self.embed(x)
        x = torch.mean(x, dim=0) # Average out the embedding values.
        x = self.linear(x)

        return x

Main process

After I run my model through a Solver with the training and testing data, I basically told the train and test functions to also return the model that's used. Then I assigned the embedding weights to a separate variable and used those as the word embeddings.

Training and testing was conducted using cross entropy loss, and each training and testing sample is of the form ([context words], target word).

def run(solver, config, vocabulary):
    for epoch in range(config.num_epochs):
        loss_train, model_train = solver.train()
        loss_test, model_test = solver.test()

        embeddings = model_train.embed.weight

I'm not sure if this is the correct way of going about extracting and using the embeddings. Is there usually another way to do this? Thanks in advance.

Output:
Yes, model_train.embed.weight will give you a torch tensor that stores the embedding weights. Note however, that this tensor also contains the latest gradients. If you don't want/need them, model_train.embed.weight.data will give you the weights only.

A more generic option is to call model_train.embed.parameters(). This will give you a generator of all the weight tensors of the layer. In general, there are multiple weight tensors in a layer and weight will give you only one of them. Embedding happens to have only one, so here it doesn't matter which option you use.


Instruction: Singularity-Container + Python + PyTorch: Why does 'import torch' work on Arch Linux host but fails on Centos 7 host?

Input:
I am trying to build a singularity container, to run a Python script on a CentOS 7-based cluster.
The container runs as expected on my host, which I also used to create the container, but fails on the cluster as soon as PyTorch is imported.

The problem can be reproduced with a container build from this minimal definition file:

debug.def:

Bootstrap: arch

%runscript
    exec /usr/bin/python3 -c 'import torch; print(torch.__version__)'

%post
    #--------------------------------------------------------------------------
    # Basic setup from
    # https://github.com/sylabs/singularity/blob/master/examples/arch/Singularity
    #--------------------------------------------------------------------------
    # Set time zone. Use whatever you prefer instead of UTC.
    ln -s /usr/share/zoneinfo/Europe/Berlin /etc/localtime

    # Set the package mirror server(s). This is only for the output image's
    # mirrorlist. `pacstrap' can only use your hosts's package mirrors.
    echo 'Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist

    pacman -Sy --noconfirm gawk sed grep

    # Set locale. Use whatever you prefer instead of en_US.
    echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen
    locale-gen
    echo 'LANG=en_US.UTF-8' > /etc/locale.conf

    pacman -S --noconfirm python python-pytorch

    pacman -S --noconfirm pacman-contrib
    paccache -r -k0

It is build with sudo singularity build debug.sif debug.def.
Both, the container and my host run on Arch Linux.

Executing the container on my host outputs the PyTorch version:

schellsn@host $ singularity run debug.sif
1.3.1

Running it on the cluster results in the following error:

schellsn@cluster tmp$ singularity run debug.sif
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/usr/lib/python3.8/site-packages/torch/__init__.py", line 81, in <module>
    from torch._C import *
ImportError: libQt5Core.so.5: cannot open shared object file: No such file or directory

I don't understand why the file is not found as it should be included in the container:

schellsn@cluster tmp$ singularity shell debug.sif
Singularity debug.sif:~/tmp> ls -l /usr/lib | grep libQt5Core
-rw-r--r--  1 root root      1166 Nov 11 23:40 libQt5Core.prl
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so -> libQt5Core.so.5.13.2
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so.5 -> libQt5Core.so.5.13.2
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so.5.13 -> libQt5Core.so.5.13.2
-rwxr-xr-x  1 root root   5275240 Nov 11 23:40 libQt5Core.so.5.13.2

I assume that the according path is not included within the search while importing and that this problem does not occur on my host because some environment setting is leaking into the container.
I also tried using the Sylabs Remote Builder but it seems that it cannot construct Arch containers (pacstrap not found within $PATH).
Trying to build the container on one of the nodes leads to the same problem; Pacstrap and pacman are not available.

I am at my wits' end and would be very grateful for any hint to explain this behavior!
Why is the shared library not found and how could that possibly be fixed?

Update #1:

Here is the content of the LD_LIBRARY_PATH environment variable (in response to @tsnowlan).

Arch Linux Host:

schellsn@host tmp$ echo $LD_LIBRARY_PATH
:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64
schellsn@host tmp$ singularity shell evpt_debug.sif
Singularity evpt_debug.sif: ~/tmp> echo $LD_LIBRARY_PATH
:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/.singularity.d/libs

CentOS 7 cluster node:

schellsn@cluster tmp$ echo $LD_LIBRARY_PATH
schellsn@cluster tmp$ singularity shell debug.sif
Singularity debug.sif:~/tmp> echo $LD_LIBRARY_PATH
/.singularity.d/libs

Update #2:

I did setup a new clean VM (also running arch) which also and rebuild the container there. This container shows the same problem; It runs on my host but not on the CentOS 7 cluster.

Output:
I was having the same issue: CentOS 7 host and Arch Linux container (Python 3.8.1 / Pytorch 1.3.1). The following link seems to have fixed my issue for now.

https://superuser.com/questions/1347723/arch-on-wsl-libqt5core-so-5-not-found-despite-being-installed

EDIT: From the link, this command worked for me

sudo strip --remove-section=.note.ABI-tag /usr/lib64/libQt5Core.so.5


Instruction: How can only giving number of channels and no height and width to my convolutional neural network work?

Input:
Hello I am a bit new to the deep learning community and I have been really fed up with how to feed in data throught a neural network. So I was doing the sentdex pytorch series and I was learning convnets. He was using the cats and dogs dataset of microsoft on kaggle. He had resized the image to 50 by 50 and turned them into grayscale. If you want to see the video to answer my question here it is -
https://pythonprogramming.net/convnet-model-deep-learning-neural-network-pytorch/

So a few thoughts came in my mind while watching the video. The input he passed is only the colour channel of the image -

At once at seeing the input he entered it came in my mind why is he only passing the number of channels which is a grayscale image. When a conv2d takes 3 inputs.

And I mean it litterally works. I tried researching a bit but no where I found a  good explaination for the input shape that is being fed in here

So I have 2 thoughts and questions about this -

So does that line mean that the convolutional neural network will
only take in an image that is grayscale and is of any height and
width and if so please tell how to limit the dimensions like that to
make our cnn only accept a input shape of (50, 50, 1).
And if not then please explain what does it mean, and how we can make
it accept any input.

Output:
Convolutional layers use the convolution operation i.e. sliding of a kernel (matrix) over the input and taking the sum of elementwise products at each position while sliding. Thus, the input dimensions will affect the output dimensions, however, it is not necessary to fix the input dimensions.
Thus, the layer can be defined as nn.Conv2d(1, 32, 5) where 1 indicates number of channels of input, 32 is number of channels of output, and 5 is the size of the kernel (it is 5x5 in this case since it is 2D).
The 32 output channels will actually mean that there will be 32 such 5x5 kernels which will be applied to the input and each output will be stacked to get an output of h x w x 32. Note that this h and w will be different than the h_in and w_in in case of not using padding, but same if you use padding.
1 input channel mentioned in the layer means that the layer will accept only single channeled inputs (which are effectively grayscale images).
If you  want to limit your CNN to use (50, 50, 1) inputs only, then you can resize the image before feeding it (you can do that using OpenCV).
Check this site for some animations of convolutions.
Update: Adding more things asked in the comments by the OP.
Yes, you can input images of any shape (I suppose they still have to be at least the size of the kernel). So, theoretically, you can input any image to a convolutional layer, but not necessarily to your CNN. That is because the CNN may have flattening operations followed by fully connected layers (nn.Linear). These flattening + fully connected will expect certain dimensions (which are fixed by you in the code), so you cannot give any input image to your CNN i.e. you have to ensure that flattening the last convolutional layer's output has dimensions equal to your first fully connected layer's dimensions.
Edit: You can actually give any sized input even for a CNN containing fully-connected layers by using a Global Average Pooling (GAP) layer to reduce the size to a fixed size irrespective of the input size. It is called Adaptive Average Pooling in PyTorch.
For example, consider this network (image attached)
In this, the convolutional kernels sizes are mentioned below the arrows, and the blue cuboids represent the output after each convolutional layer. At the end, there are fully connected layers (boxes with circles) which have fixed dimensions. So, the last convolutional layer output has dimensions 66256 = 9216, which is also the dimension of the first fully connected layer.
So, basically, you design your network such that the last convolutional output flattened has same dimensions as the first fully connected layer. Note that there are some networks called Fully Convolutional Networks (FCNs) which don't use these fully connected layers and thus are input size independent. The network design and choice of layers depends on your application.


Instruction: Pytorch Categorical Cross Entropy loss function behaviour

Input:
I have question regarding the computation made by the Categorical Cross Entropy Loss from Pytorch.
I have made this easy code snippet and because I use the argmax of the output tensor as the targets, I cannot understand why the loss is still high.

import torch
import torch.nn as nn
ce_loss = nn.CrossEntropyLoss()
output = torch.randn(3, 5, requires_grad=True)
targets = torch.argmax(output, dim=1)
loss = ce_loss(outputs, targets)
print(loss)

Thanks for the help understanding it.
Best regards
Jerome

Output:
So here is a sample data from your code with the output, label and loss having the following values

outputs =  tensor([[ 0.5968, -0.8249,  1.5018,  2.7888, -0.6125],
                   [-1.1534, -0.4921,  1.0688,  0.2241, -0.0257],
                   [ 0.3747,  0.8957,  0.0816,  0.0745,  0.2695]], requires_grad=True)requires_grad=True)

labels = tensor([3, 2, 1])
loss = tensor(0.7354, grad_fn=<NllLossBackward>)

So let's examine the values,

If you compute the softmax output of your logits (outputs), using something like this torch.softmax(outputs,axis=1) you will get

probs = tensor([[0.0771, 0.0186, 0.1907, 0.6906, 0.0230],
                [0.0520, 0.1008, 0.4801, 0.2063, 0.1607],
                [0.1972, 0.3321, 0.1471, 0.1461, 0.1775]], grad_fn=<SoftmaxBackward>)

So these will be your prediction probabilities.

Now cross-entropy loss is nothing but a combination of softmax and negative log likelihood loss. Hence, your loss can simply be computed using

loss = (torch.log(1/probs[0,3]) +  torch.log(1/probs[1,2]) + torch.log(1/probs[2,1])) / 3

, which is the average of the negative log of the probabilities of your true labels. The above equation evaluates to 0.7354, which is equivalent to the value returned from the nn.CrossEntropyLoss module.


Instruction: Sparse vectors for training data

Input:
I have a training data like this:

x_train = np.random.randint(100, size=(1000, 25))

where each row is a sample and thus we have 1000 samples.

Now I need to have the training data such that for each of the sample/row there can be at max 3 non-zero elements out of 25.

Can you all please suggest how I can implement that? Thanks!

Output:
I am assuming that you want to turn a majority of your data into zeros, except that 0 to 3 non-zero elements are retained (randomly) for each row. If this is the case, a possible way to do this is as follows.

Code

import numpy as np
max_ = 3
nrows = 1000
ncols = 25

np.random.seed(7)

X = np.zeros((nrows,ncols))

data = np.random.randint(100, size=(nrows, ncols))

# number of max non-zeros to be generated for each column
vmax = np.random.randint(low=0, high=4, size=(nrows,))

for i in range(nrows):

  if vmax[i]>0:
      #index for setting non-zeros
      col = np.random.randint(low=0, high=ncols, size=(1,vmax[i]))

      #set non-zeros elements
      X[i][col] = data[i][col]

print(X)

Output

[[ 0. 68. 25. ...  0.  0.  0.]
 [ 0.  0.  0. ...  0.  0.  0.]
 [ 0.  0.  0. ...  0.  0.  0.]
 ...
 [ 0.  0.  0. ...  0.  0.  0.]
 [88.  0.  0. ...  0.  0.  0.]
 [ 0.  0.  0. ...  0.  0.  0.]]


Instruction: What sort of loss function should I use this multi-class multi-label(?) problem?

Input:
In my experiment I am trying to train a neural network to detect if patients exhibit symptom A, B, C, D. My data consists of different angled photos of each patient along with whether or not they have symptom A, B, C, D.

Right now in, pytoch, I am using MSELoss and calculating my test error as the total number of correct classifications out of the total number of classifications. I'm guessing this is too naive and even inappropriate.

An example of a test error computation would be like this:
Suppose we have 2 patients with two images each of them. Then there would be 16 total classifications (1 for whether patient 1 has symptom A, B, C, D in photo 1, etc). And if the model correctly predicted that in photo 1 patient 1 exhibited symptom A then that would add 1 to the total number of correct classifications.

Output:
I suggest to use binary-crossentropy in multi-class multi-label classifications. This may seem counterintuitive for multi-label classification, but keep in mind that the goal here is to treat each output label as an independent distribution (or class).

In pytorch you can use torch.nn.BCELoss(weight=None, size_average=None, reduce=None, reduction='mean'). This creates a criterion that measures the Binary Cross Entropy between the target and the output.


Instruction: How to get the device type of a pytorch module conveniently?

Input:
I have to stack some my own layers on different kinds of pytorch models with different devices.

E.g. A is a cuda model and B is a cpu model (but I don't know it before I get the device type). Then the new models are C and D respectively, where

class NewModule(torch.nn.Module):
    def __init__(self, base):
        super(NewModule, self).__init__()
        self.base = base
        self.extra = my_layer() # e.g. torch.nn.Linear()

    def forward(self,x):
        y = self.base(x)
        z = self.extra(y)
        return z

...

C = NewModule(A) # cuda
D = NewModule(B) # cpu

However I must move base and extra to the same device, i.e. base and extra of C are cuda models and D's are cpu models. So I tried this __inin__:

def __init__(self, base):
    super(NewModule, self).__init__()
    self.base = base
    self.extra = my_layer().to(base.device)

Unfortunately, there's no attribute device in torch.nn.Module(raise AttributeError).

What should I do to get the device type of base? Or any other method to make base and extra to be on the same device automaticly even the structure of base is unspecific?

Output:
This question has been asked many times (1, 2). Quoting the reply from a PyTorch developer:

That’s not possible. Modules can hold parameters of different types on different devices, and so it’s not always possible to unambiguously determine the device.

The recommended workflow (as described on PyTorch blog) is to create the device object separately and use that everywhere. Copy-pasting the example from the blog here:

# at beginning of the script
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

...

# then whenever you get a new Tensor or Module
# this won't copy if they are already on the desired device
input = data.to(device)
model = MyModule(...).to(device)

Do note that there is nothing stopping you from adding a .device property to the models.

As mentioned by Kani (in the comments), if the all the parameters in the model are on the same device, one could use next(model.parameters()).device.


Instruction: Pass Python object to C and back again

Input:
I'm trying to get to grips with using embedded Python from a C++ application. Specifically I want my C++ to launch some PyTorch code.

I am making some initialization function in Python to perform the device (CPU or GPU) discovery and would like to pass this back to the C++ code. The C++ will call another Python function for inference which is when the C++ will pass the device to Python.

        pFunc_init = PyObject_GetAttrString(pModule, "torch_init");
        if (pFunc_init && PyCallable_Check(pFunc_init)) {
            pValue_device = PyObject_CallObject(pFunc_init, pArgs);

            if (pArgs != NULL)
                Py_DECREF(pArgs);

            if (pValue_device != NULL) {
                pFunc_infer = PyObject_GetAttrString(pModule, "torch_infer");
                if (pFunc_infer && PyCallable_Check(pFunc_infer)) {
                    //
                    // TODO put object pValue_device into pArgs_device.
                    //
                    pValue_infer = PyObject_CallObject(pFunc_infer, pArgs_device);
                    if (pValue_infer != NULL) {
                        printf("Result pValue_infer: %ld\n", PyLong_AsLong(pValue_infer));
                        Py_DECREF(pValue_infer);
                    }
                }
                Py_DECREF(pValue_device);
            }
            else {
                Py_DECREF(pFunc_init);
                Py_DECREF(pModule);
                PyErr_Print();
                fprintf(stderr, "Call failed\n");
                return 1;
            }
        }

The TODO marks where I would like to put this code. With simple Python objects I think I know what I need but how to deal with this custom Python object?

Output:
You can define a Python function "detect_device" which returns a string say "cuda" or "cpu". After that in your C++ code, you can do something like this.

PyObject *detect_device, *pArgsDevice;
detect_device  = PyObject_GetAttrString(pModule, "detect_device");
deviceObject = PyObject_CallObject(detect_device, NULL);

pArgsDevice = NULL;
pArgsDevice = PyTuple_New(1);
PyTuple_SetItem(pArgsDevice, 0, deviceObject);

PS: Wrote the answer in hurry due to some urgency. Will add explanation soon, but I think if you understand the code that you have written, you would be able to understand this. Letme know in comments about your progress.