Instruction: Different output from Libtorch C++ and pytorch

Input:
I'm using the same traced model in pytorch and libtorch but I'm getting different outputs.
Python Code:
import cv2
import numpy as np
import torch
import torchvision
from torchvision import transforms as trans

# device for pytorch
device = torch.device('cuda:0')

torch.set_default_tensor_type('torch.cuda.FloatTensor')

model = torch.jit.load("traced_facelearner_model_new.pt")
model.eval()

# read the example image used for tracing
image=cv2.imread("videos/example.jpg")

test_transform = trans.Compose([
        trans.ToTensor(),
        trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
    ])

resized_image = cv2.resize(image, (112, 112))

tens = test_transform(resized_image).to(device).unsqueeze(0)
output = model(tens)
print(output)

C++ Code:
#include <iostream>
#include <algorithm>
#include <opencv2/opencv.hpp>
#include <torch/script.h>

int main()
{
    try
    {
        torch::jit::script::Module model = torch::jit::load("traced_facelearner_model_new.pt");
        model.to(torch::kCUDA);
        model.eval();

        cv::Mat visibleFrame = cv::imread("example.jpg");

        cv::resize(visibleFrame, visibleFrame, cv::Size(112, 112));
        at::Tensor tensor_image = torch::from_blob(visibleFrame.data, { 1, visibleFrame.rows,
                                                    visibleFrame.cols, 3 }, at::kByte);
        tensor_image = tensor_image.permute({ 0, 3, 1, 2 });
        tensor_image = tensor_image.to(at::kFloat);

        tensor_image[0][0] = tensor_image[0][0].sub(0.5).div(0.5);
        tensor_image[0][1] = tensor_image[0][1].sub(0.5).div(0.5);
        tensor_image[0][2] = tensor_image[0][2].sub(0.5).div(0.5);

        tensor_image = tensor_image.to(torch::kCUDA);
        std::vector<torch::jit::IValue> input;
        input.emplace_back(tensor_image);
        // Execute the model and turn its output into a tensor.
        auto output = model.forward(input).toTensor();
        output = output.to(torch::kCPU);
        std::cout << "Embds: " << output << std::endl;

        std::cout << "Done!\n";
    }
    catch (std::exception e)
    {
        std::cout << "exception" << e.what() << std::endl;
    }
}

The model gives (1x512) size output tensor as shown below.
Python output
tensor([[-1.6270e+00, -7.8417e-02, -3.4403e-01, -1.5171e+00, -1.3259e+00,

-1.1877e+00, -2.0234e-01, -1.0677e+00, 8.8365e-01, 7.2514e-01,

2.3642e+00, -1.4473e+00, -1.6696e+00, -1.2191e+00, 6.7770e-01,

...

-7.1650e-01, 1.7661e-01]], device=‘cuda:0’,
grad_fn=)

C++ output
Embds: Columns 1 to 8 -84.6285 -14.7203 17.7419 47.0915 31.8170 57.6813 3.6089 -38.0543

Columns 9 to 16 3.3444 -95.5730 90.3788 -10.8355 2.8831 -14.3861 0.8706 -60.7844

...

Columns 505 to 512 36.8830 -31.1061 51.6818 8.2866 1.7214 -2.9263 -37.4330 48.5854

[ CPUFloatType{1,512} ]

Using

Pytorch 1.6.0
Libtorch 1.6.0
Visual studio 2019
Windows 10
Cuda 10.1

Output:
before the final normalization, you need to scale your input to the range 0-1 and then carry on the normalization you are doing. convert to float and then divide by 255 should get you there. Here is the snippet I wrote, there might be some syntaax errors, that should be visible.
Try this :
#include <iostream>
#include <algorithm>
#include <opencv2/opencv.hpp>
#include <torch/script.h>

int main()
{
    try
    {
        torch::jit::script::Module model = torch::jit::load("traced_facelearner_model_new.pt");
        model.to(torch::kCUDA);

        cv::Mat visibleFrame = cv::imread("example.jpg");

        cv::resize(visibleFrame, visibleFrame, cv::Size(112, 112));
        at::Tensor tensor_image = torch::from_blob(visibleFrame.data, {  visibleFrame.rows,
                                                    visibleFrame.cols, 3 }, at::kByte);

        tensor_image = tensor_image.to(at::kFloat).div(255).unsqueeze(0);
        tensor_image = tensor_image.permute({ 0, 3, 1, 2 });
        ensor_image.sub_(0.5).div_(0.5);

        tensor_image = tensor_image.to(torch::kCUDA);
        // Execute the model and turn its output into a tensor.
        auto output = model.forward({tensor_image}).toTensor();
        output = output.cpu();
        std::cout << "Embds: " << output << std::endl;

        std::cout << "Done!\n";
    }
    catch (std::exception e)
    {
        std::cout << "exception" << e.what() << std::endl;
    }
}

I don't have access to a system to run this so if you face anything comment below.


Instruction: Pytorch is really slow and uses a lot of GPU memory when used in Starlette with WEB_CONCURRENCY > 1

Input:
I am trying to build an API that uses a Pytorch model. However, as soon as I increase WEB_CONCURRENCY to something above 1, it creates substantially more threads than expected and slows down by a lot, even when sending a single request.
Example code:
api.sh
export WEB_CONCURRENCY=2

python api.py

api.py
from starlette.applications import Starlette
from starlette.responses import UJSONResponse
from starlette.middleware.gzip import GZipMiddleware
from mymodel import Model

model = Model()
app = Starlette(debug=False)
app.add_middleware(GZipMiddleware, minimum_size=1000)

@app.route('/process', methods=['GET', 'POST', 'HEAD'])
async def add_styles(request):
    if request.method == 'GET':
        params = request.query_params
    elif request.method == 'POST':
        params = await request.json()
    elif request.method == 'HEAD':
        return UJSONResponse([], headers=response_header)

    print('===Request body===')
    print(params)

    model_output = model(params.get('data', [])) # It is very simplified. Inside there are
                                                 # many things that are happening, which
                                                 # involve file reading/writing
                                                 # and spawning processes with `popen` that
                                                 # do even more processing. But I don't
                                                 # think that should be an issue here.

    return model_output

if __name__ == '__main__':
    uvicorn.run('api:app', host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))

When WEB_CONCURRENCY=1 in api.sh, there is only 1 python process seen when nvidia-smi is ran and model uses 1.2GB or VRAM. Request takes ~0.7s
When WEB_CONCURRENCY=2 in api.sh, there can be upwards of 8 python processes seen in nvidia-smi and they will use upwards of ~8GB of VRAM. Then one single request can take up to 3s, if you're lucky and don't get an out of memory error.
I am using Python3.8
Why isn't Pytorch using the expected VRAM of 2.4GB when WEB_CONCURRENCY=2? And why is it slowing down so much?

Output:
If anyone else stumbles upon this issue, just use gunicorn. It uses separate threads/processes, so there's no internal conflict going on.
So instead of running it with: python api.py, just run with: gunicorn -w 2 api:app -k uvicorn.workers.UvicornWorker


Instruction: Pytorch, get rid of a for loop when adding permutation of one vector to entries of a matrix?

Input:
I'm trying to implement this paper, and stuck with this simple step. Although this is to do with attention, the thing I'm stuck with is just how to implement a permutation of a vector added to a matrix without using for loops.
The attention scores have a learned bias vector added to them, the theory is that it encodes relative position (j-i) of the two tokens the score represents
so alpha is a T x T matrix,T depends on the batch being forwarded, and B is a learned bias vector whose length has to be fixed and as large as 2T. My current implementation which I believe does what the paper suggests is:
    def __init__(...):
       ...
        self.bias = torch.nn.Parameter(torch.randn(config.n),requires_grad = True)
        stdv = 1. / math.sqrt(self.bias.data.size(0))
        self.bias.data.uniform_(-stdv, stdv)
     def forward(..)
        ...
        #n = 201  (2* max_seq_len + 1)

        B_matrix = torch.zeros(self.T, self.T) # 60 x 60
        for i in range(self.T):
          B_matrix[i] = self.bias[torch.arange(start=n//2-i, end=n//2-i+T)])]

        attention_scores = attention_scores + B_matrix.unsqueeze(0)
        # 64 x 60 x 60
        ...

This is the only relevant part
B_matrix = torch.zeros(self.T, self.T) # 60 x 60
        for i in range(self.T):
          B_matrix[i] = self.bias[torch.arange(start=n//2-i, end=n//2-i+T)])]

basically trying to not use a for loop to go over each row.
but I know this must be really inefficient, and costly when this model is very large. I'm doing an explicit for loop over each row to get a permutation of the learned bias vector.
Can anyone help me out with a better way, through smart broadcasting perhaps?
After thinking about it, I don't need to instantiate a zero matrix, but still can't get rid of the for loop? and can't use gather as the B_matrix is a different size than a tiled b vector.
functor = lambda i : bias[torch.arange(start=n//2-i, end=n//2-i+T)]
B_matrix = torch.stack([functor(i) for i in torch.arange(T)])

Output:
I couldn't figure out what n was supposed to be in your code but I think the following example using torch.meshgrid provides what you're looking for.
Supposing
n, m = 10, 20   # arbitrary
a = torch.randn(n, m)
b = torch.randn(n + m)

then
for i in range(n):
    for j in range(m):
        a[i, j] = a[i, j] + b[n - i + j]

is equivalent to
ii, jj = torch.meshgrid(torch.arange(n), torch.arange(m))
a = a + b[n - ii + jj]

though the latter is an out-of-place operation, which is usually a good thing. If you actually wanted an in-place operation then replace a = with a[...] =.
Note that this is an example of integer array indexing where we index b using a tensor that is the same shape as a.


Instruction: How to avoid recalculating a function when we need to backpropagate through it twice?

Input:
In PyTorch, I want to do the following calculation:
l1 = f(x.detach(), y)
l1.backward(retain_graph=True)
l2 = -1*f(x, y.detach())
l2.backward()

where f is some function, and x and y are tensors that require gradient. Notice that x and y may both be the results of previous calculations which utilize shared parameters (for example, maybe x=g(z) and y=g(w) where g is an nn.Module).
The issue is that l1 and l2 are both numerically identical, up to the minus sign, and it seems wasteful to repeat the calculation f(x,y) twice. It would be nicer to be able to calculate it once, and apply backward twice on the result. Is there any way of doing this?
One possibility is to manually call autograd.grad and update the w.grad field of each nn.Parameter w. But I'm wondering if there is a more direct and clean way to do this, using the backward function.

Output:
I took this answer from here.
We can calculate f(x,y) once, without detaching neither x or y, if we ensure that we we multiply by -1 the gradient flowing through x. This can be done using register_hook:
x.register_hook(lambda t: -t)
l = f(x,y)
l.backward()

Here is code demonstrating that this works:
import torch

lin = torch.nn.Linear(1, 1, bias=False)
lin.weight.data[:] = 1.0
a = torch.tensor([1.0])
b = torch.tensor([2.0])
loss_func = lambda x, y: (x - y).abs()

# option 1: this is the inefficient option, presented in the original question
lin.zero_grad()
x = lin(a)
y = lin(b)
loss1 = loss_func(x.detach(), y)
loss1.backward(retain_graph=True)
loss2 = -1 * loss_func(x, y.detach())  # second invocation of `loss_func` - not efficient!
loss2.backward()
print(lin.weight.grad)

# option 2: this is the efficient method, suggested in this answer.
lin.zero_grad()
x = lin(a)
y = lin(b)
x.register_hook(lambda t: -t)
loss = loss_func(x, y)  # only one invocation of `loss_func` - more efficient!
loss.backward()
print(lin.weight.grad)  # the output of this is identical to the previous print, which confirms the method

# option 3 - this should not be equivalent to the previous options, used just for comparison
lin.zero_grad()
x = lin(a)
y = lin(b)
loss = loss_func(x, y)
loss.backward()
print(lin.weight.grad)


Instruction: Mask certain indices for every entry in a batch, when using torch.max()

Input:
I am incremently sampling a batch of size torch.Size([n, 8]).
I also have a list valid_indices of length n which contains tuples of indices that are valid for each entry in the batch.
For instance valid_indices[0] may look like this: (0,1,3,4,5,7) , which suggests that indices 2 and 6 should be excluded from the first entry in batch along dim 1.
Particularly I need to exclude these values for when I use torch.max(batch, dim=1, keepdim=True).
Indices to be excluded (if any) may differ from entry to entry within the batch.
Any ideas? Thanks in advance.

Output:
I assume that you are getting the good old
IndexError: too many indices for tensor of dimension 1

error when you use your tuple indices directly on the tensor.
At least that was the error that I was able to reproduce when I execute the following line
 t[0][valid_idx0]

Where t is a random tensor with size (10,8) and valid_idx0 is a tuple with 4 elements.
However, same line works just fine when you convert your tuple to a list as following
 t[0][list(valid_idx0)]

 >>> tensor([0.1847, 0.1028, 0.7130, 0.5093])

But when it comes to applying these indices to 2D tensors, things get a bit different, since we need to preserve the structure of our tensor for batch processing.
Therefore, it would be reasonable to convert our indices to mask arrays.
Let's say we have a list of tuples valid_indices at hand. First thing will be converting it to a list of lists.
valid_idx_list = [list(tup) for tup in valid_indices]

Second thing will be converting them to mask arrays.
masks = np.zeros((t.size()))
for i, indices in enumerate(valid_idx_list):
  masks[i][indices] = 1

Done. Now we can apply our mask and use the torch.max on the masked tensor.
torch.max(t*masks)

Kindly see the colab notebook that I've used to reproduce the problem.
https://colab.research.google.com/drive/1BhKKgxk3gRwUjM8ilmiqgFvo0sfXMGiK?usp=sharing


Instruction: How to manually implement padding for pytorch convolutions

Input:
I'm trying to port some pytorch code to tensorflow 2.0 and am having difficulty figuring out how to translate the convolution functions between the two. The way both libraries deal with padding is the sticking point. Basically, I'd like to understand how I can manually produce the padding that pytorch does under the hood so that I can translate that to tensorflow.
The code below works if I don't do any padding, but I can't figure out how to make the two implementations match once any padding is added.
output_padding = SOME NUMBER
padding = SOME OTHER NUMBER
strides = 128

tensor = np.random.rand(2, 258, 249)
filters = np.random.rand(258, 1, 256)

out_torch = F.conv_transpose1d(
    torch.from_numpy(tensor).float(),
    torch.from_numpy(filters).float(),
    stride=strides,
    padding=padding,
    output_padding=output_padding)

def pytorch_transpose_conv1d(inputs, filters, strides, padding, output_padding):
    N, L_in = inputs.shape[0], inputs.shape[2]
    out_channels, kernel_size = filters.shape[1], filters.shape[2]
    time_out = (L_in - 1) * strides - 2 * padding + (kernel_size - 1) + output_padding + 1
    padW = (kernel_size - 1) - padding

    # HOW DO I PAD HERE TO GET THE SAME OUTPUT AS IN PYTORCH
    inputs = tf.pad(inputs, [(?, ?), (?, ?), (?, ?)])

    return tf.nn.conv1d_transpose(
        inputs,
        tf.transpose(filters, perm=(2, 1, 0)),
        output_shape=(N, out_channels, time_out),
        strides=strides,
        padding="VALID",
        data_format="NCW")

out_tf = pytorch_transpose_conv1d(tensor, filters, strides, padding, output_padding)
assert np.allclose(out_tf.numpy(), out_torch.numpy())

Output:
Padding

To translate the convolution and transpose convolution functions (with padding padding) between the Pytorch and Tensorflow we need to understand first F.pad() and tf.pad() functions.
torch.nn.functional.pad(input, padding_size, mode='constant', value=0):

padding size: The padding size by which to pad some dimensions of input are described starting from the last dimension and moving forward.
to pad only the last dimension of the input tensor, then pad has the form (padding_left, padding_right)
to pad the last 3 dimensions, (padding_left, padding_right, padding_top, padding_bottom, padding_front,padding_back)

tensorflow.pad(input, padding_size, mode='CONSTANT',name=None,constant_values=0)

padding_size: is an integer tensor with shape [n, 2], where n is the rank of the tensor. For each dimension D of input, paddings[D, 0] indicates how many values to add before the contents of tensor in that dimension, and paddings[D, 1] indicates how many values to add after the contents of tensor in that dimension.

Here's table representing F.pad and tf.pad equivalents along with output tensor for the input tensor
[[[1, 1], [1, 1]]] which is of shape (1, 2, 2)

Padding in Convolution

Let's now move to PyTorch padding in Convolution layers

F.conv1d(input, ..., padding, ...):

padding controls the amount of implicit paddings on both sides for padding number of points.
padding=(size) applies F.pad(input, [size, size]) i.e padding last dimension with (size, size) equivalent to tf.pad(input, [[0, 0], [0, 0], [size, size]])

F.conv2d(input, ..., padding, ...):

padding=(size) applies F.pad(input, [size, size, size, size]) i.e padding last 2 dimensions with (size, size) equivalent to tf.pad(input, [[0, 0], [size, size], [size, size]])
padding=(size1, size2) applies F.pad(input, [size2,  size2, size1, size1]) which is equivalent to tf.pad(input, [[0, 0], [size1, size1], [size2, size2]])

Padding in Transpose Convolution

PyTorch padding in Transpose Convolution layers

F.conv_transpose1d(input, ..., padding, output_padding, ...):

dilation * (kernel_size - 1) - padding padding will be added to both sides of each dimension in the input.
Padding in transposed convolutions can be seen as allocating fake outputs that will be removed
output_padding controls the additional size added to one side of the output shape
Check this to understand what exactly happens during transpose convolution in pytorch.
Here's the formula to calculate output size of transpose convolution:

output_size = (input_size - 1)stride + (kerenel_size - 1) + 1 + output_padding - 2padding

Codes

Transpose Convolution
import torch
import torch.nn as nn
import torch.nn.functional as F
import tensorflow as tf
import numpy as np

# to stop tf checkfailed error not relevent to actual code
import os
os.environ["CUDA_DEVICE_ORDER"]    = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "1"

def tconv(tensor, filters, output_padding=0, padding=0, strides=1):
    '''
    tensor         : input tensor of shape (batch_size, channels, W) i.e (NCW)
    filters        : input kernel of shape (in_ch, out_ch, kernel_size)
    output_padding : single number must be smaller than either stride or dilation
    padding        : single number should be less or equal to ((valid output size + output padding) // 2)
    strides        : single number
    '''
    bs, in_ch, W = tensor.shape
    in_ch, out_ch, k_sz = filters.shape

    out_torch = F.conv_transpose1d(torch.from_numpy(tensor).float(),
                                   torch.from_numpy(filters).float(),
                                   stride=strides, padding=padding,
                                   output_padding=output_padding)
    out_torch = out_torch.numpy()

    # output_size = (input_size - 1)*stride + (kerenel_size - 1) + 1 + output_padding - 2*padding
    # valid out size -> padding=0, output_padding=0
    # -> valid_out_size =  (input_size - 1)*stride + (kerenel_size - 1) + 1
    out_size  = (W - 1)*strides + (k_sz - 1) + 1

    # input shape -> (batch_size, W, in_ch) and filters shape -> (kernel_size, out_ch, in_ch) for tf conv
    valid_tf  = tf.nn.conv1d_transpose(np.transpose(tensor, axes=(0, 2, 1)),
                                       np.transpose(filters, axes=(2, 1, 0)),
                                       output_shape=(bs, out_size, out_ch),
                                       strides=strides, padding='VALID',
                                       data_format='NWC')
    # output padding
    tf_outpad = tf.pad(valid_tf, [[0, 0], [0, output_padding], [0, 0]])
    # NWC to NCW
    tf_outpad = np.transpose(tf_outpad, (0, 2, 1))

    # padding -> input, begin, shape -> remove `padding` elements on both side
    out_tf    = tf.slice(tf_outpad, [0, 0, padding], [bs, out_ch, tf_outpad.shape[2]-2*padding])

    out_tf    = np.array(out_tf)

    print('output size(tf, torch):', out_tf.shape, out_torch.shape)
    # print('out_torch:\n', out_torch)
    # print('out_tf:\n', out_tf)
    print('outputs are close:', np.allclose(out_tf, out_torch))

tensor  = np.random.rand(2, 1, 7)
filters = np.random.rand(1, 2, 3)
tconv(tensor, filters, output_padding=2, padding=5, strides=3)

Results
>>> tensor  = np.random.rand(2, 258, 249)
>>> filters = np.random.rand(258, 1, 7)
>>> tconv(tensor, filters, output_padding=4, padding=9, strides=6)
output size(tf, torch): (2, 1, 1481) (2, 1, 1481)
outputs are close: True

Some useful links:

pytorch 'SAME' convolution

How pytorch transpose conv works


Instruction: Gensim's word2vec has a loss of 0 from epoch 1?

Input:
I am using the Word2vec module of Gensim library to train a word embedding, the dataset is 400k sentences with 100k unique words (its not english)
I'm using this code to monitor and calculate the loss :
class MonitorCallback(CallbackAny2Vec):
    def __init__(self, test_words):
        self._test_words = test_words

    def on_epoch_end(self, model):
        print("Model loss:", model.get_latest_training_loss())  # print loss
        for word in self._test_words:  # show wv logic changes
            print(model.wv.most_similar(word))

monitor = MonitorCallback(["MyWord"])  # monitor with demo words

w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, window=W2V_WINDOW, min_count=W2V_MIN_COUNT  , callbacks=[monitor])

w2v_model.build_vocab(tokenized_corpus)

words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print("Vocab size", vocab_size)

print("[*] Training...")

# Train Word Embeddings
w2v_model.train(tokenized_corpus, total_examples=len(tokenized_corpus), epochs=W2V_EPOCH)

The problem is from epoch 1 the loss is 0 and the vector of the monitored words dont change at all!
[*] Training...
Model loss: 0.0
Model loss: 0.0
Model loss: 0.0
Model loss: 0.0

so what is the problem here? is this normal?  the tokenized corpus is a list of lists that are something like tokenized_corpus[0] = [ "word1" , "word2" , ...]
I googled and seems like some of the old versions of gensim had problem with calculating loss function, but they are from almost a year ago and it seems like it should be fixed right now?
I tried the code provided in the answer of this question as well but still the loss is 0 :
Loss does not decrease during training (Word2Vec, Gensim)
EDIT1 : after adding compute_loss=True, the loss shows up, but it keeps going higher and higher, and the top similar words and their similarity doesn't change at all :
Model loss: 2187903.5
Model loss: 3245492.0
Model loss: 4103624.5
Model loss: 4798541.0
Model loss: 5413940.0
Model loss: 5993822.5
Model loss: 6532631.0
Model loss: 7048384.5
Model loss: 7547147.0

Output:
The top issue with your code is that you haven't used the Word2Vec initialization parameter necessary to toggle loss-tracking on: compute_loss=True
(See 'parameters' section of https://radimrehurek.com/gensim/models/word2vec.html#gensim.models.word2vec.Word2Vec )
Even with that fix, the loss-reporting is still quite buggy (as of gensim-3.8.3 & this writing in August 2020):

it's not the per-epoch total, or per-example average, one might expect. (So if you need that, as a workaround, your callback should be remembering the last value and computing the delta, or resetting the internal counter to 0.0, each epoch's end.)
it definitely loses precision in larger training runs, eventually becoming useless. (This may not be an issue for you.)
it might lose some tallies due to multithreaded value-overwriting. (This may not be a practical issue for you, depending on why you're consulting the loss value.)


Instruction: Resnet implementation: forward() takes 1 positional argument but 2 were given

Input:
I wrote this code and when I run it I get the following error: forward() takes 1 positional argument but 2 were given. As far as I know, I am passing only one argument to forward().

ResNet is a basic residual block

class ResNet(nn.Module):
    def __init__(self, in_channels, mid_channels, mid2_channels ,out_channels):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels,mid_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv1_bn = nn.BatchNorm2d(mid_channels)
        self.conv2 = nn.Conv2d(mid_channels,mid2_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv2_bn = nn.BatchNorm2d(mid2_channels)
        self.conv3 = nn.Conv2d(mid2_channels,out_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv3_bn = nn.BatchNorm2d(out_channels)
        if (in_channels != out_channels):
            self.conv_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size = 1, stride = 1, padding = 0 )

    def forward(self, X):
         X_shortcut = X
        X = F.relu(self.conv1(X))
        X =  self.conv1_bn(X)
        X = F.relu(self.conv2(X))
        X =  self.conv2_bn(X)
        X = F.relu(self.conv2(X))
        X =  self.conv2_bn(X)
        if (in_channels == out_channels):
            X = self.conv3(X) + X_shortcut
        else:
            X = self.conv3(X) + self.conv_shortcut(X_shortcut)
        X = self.conv3_bn(F.relu(x))
        return X

This the method for generating a model using the given layers.

class TotalNet(nn.Module):
    def __init__(self, Layers):
        super().__init__()
        self.hidden = nn.ModuleList()
        self.hidden.append(nn.BatchNorm2d(1))
        for i in range(0,len(Layers)-1,3):
            in_channels, mid_channels, mid2_channels, out_channels = Layers[i:(i+4)]
            self.hidden.append(ResNet(in_channels, mid_channels, mid2_channels, out_channels))
            self.hidden.append(nn.Flatten())

    def forward(self, X):
        X = self.hidden(X)
        return X

the following is how I am calling the function:

test = TotalNet([9,2,9,9,9,9,9,9,9,9])
a = torch.rand((1,9,9), dtype = torch.float32)
test(a)

Output:
I realized that I was passing the X to the nn.ModuleList. This is incorrect that the right way would be to apply X to the elements of nn.ModuleList and updating the values of X.
In other words, the forward function of TotalNet should be the following:
for operation in self.hidden:
            X = operation(X)
        return X


Instruction: Pytorch: after embedding layer, Unable to get repr for

Input:
I am new to PyTorch and trying to reproduce the project: https://github.com/eXascaleInfolab/ActiveLink
However, errors occur in the feedforward() which has been bothering me for days, here is part of the code (for complete code of the model, see https://github.com/eXascaleInfolab/ActiveLink/blob/master/models.py please):
def forward(self, e1, rel, batch_size=None, weights=None):
......
        e1_embedded = self.emb_e(e1).view(-1, 1, 10, 20)
        rel_embedded = self.emb_rel(rel).view(-1, 1, 10, 20)
        stacked_inputs = torch.cat([e1_embedded, rel_embedded], 2)  # out: (128L, 1L, 20L, 20L)

That gives me the error (I am using GPU):
THCudaCheck FAIL file=/pytorch/aten/src/THC/generic/THCTensorMath.cu line=196 error=710 : device-side assert triggered
Traceback (most recent call last):
  File "main.py", line 147, in <module>
    main()
  File "main.py", line 136, in main
    model = run_meta_incremental(config, model, train_batcher, test_rank_batcher)
  File "/home/yonghui/yt/meta_incr_training.py", line 158, in run_meta_incremental
    g = run_inner(config, model, task)
  File "/home/yonghui/yt/meta_incr_training.py", line 120, in run_inner
    pred = model.forward(e1, rel)
  File "/home/yonghui/yt/models.py", line 136, in forward
    stacked_inputs = torch.cat([e1_embedded, rel_embedded], 2)
RuntimeError: cuda runtime error (710) : device-side assert triggered at /pytorch/aten/src/THC/generic/THCTensorMath.cu:196
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.

I use Debugger in an attempt to find out where goes wrong:
Before e1 and rel are embedded, they are both tensors in int64 with the shape of torch.Size([128, 1]).
e1 can be embedded as normal, converting into torch.float32 and torch.Size([128, 1, 10, 20]). However, after rel passed the embedding layer of emb_rel, Debugger shows all tenros as Unable to get repr for <class 'torch.Tensor'>.
What's going on? How can I fix that? Thank you for any possible help!!

Output:
This issue is solved by using the debugger and checking the input tensor.
After checking the tensors before embedding, I find that some elements exceed the range, especially for the case where the index starting from 0.


Instruction: Value error while converting tensor to numpy array

Input:
I'm using the following code to extract the features from image.
def ext():
    imgPathList = glob.glob("images/"+"*.JPG")
    features = []
    for i, path in enumerate(tqdm(imgPathList)):
        feature = get_vector(path)
        feature = feature[0] / np.linalg.norm(feature[0])
        features.append(feature)
        paths.append(path)
    features = np.array(features, dtype=np.float32)
    return features, paths

However, the above code throws the following error,
 features = np.array(features, dtype=np.float32)
ValueError: only one element tensors can be converted to Python scalars

How can I be able to fix it?

Output:
The error says that your features variable is a list which contains multi dimensional values which cant be converted to tensor, because .append is converting the tensors to list, So some workaround is to use concatenation function of torch as torch.cat() (read here) instead of append method. I tried to replicate the solution with toy example.
I am assuming that features contain 2D tensor
import torch
for i in range(1,11):
    alpha = torch.rand(2,2)
    if i<2:
        beta = alpha #will concatenate second sample
    else:
        beta = torch.cat((beta,alpha),0)

import numpy as np

features = np.array(beta, dtype=np.float32)


Instruction: PyTorch find keypoints: output nodes to be in a range and negative loss

Input:
I am beginner in deep learning.
I am using this dataset and I want my network to detect keypoints of a hand.
How can I make my output layer's nodes to be in range [-1, 1] (range of normalized 2D points)?
Another problem is when I train for more than 1 epoch the loss gets negative values
criterion: torch.nn.MultiLabelSoftMarginLoss() and optimizer: torch.optim.SGD()
Here u can find my repo
net = nnModel.Net()
net = net.to(device)
criterion = nn.MultiLabelSoftMarginLoss()
optimizer = optim.SGD(net.parameters(), lr=learning_rate)
lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=decay_rate)

Output:
You can use the Tanh activation function, since the image of the function lies in [-1, 1].
The problem of predicting key-points in an image is more of a regression problem than a classification problem (especially if you're making your model outputs + targets fall within a continuous interval). Therefore, I suggest you use the L2 Loss.
In fact, it could be a good exercise for you to determine which loss function that is appropriate for regression problems provides the lowest expected generalization error using cross-validation. There's several such functions available in PyTorch.


Instruction: How to change the learning rate in PyTorch (1.6)

Input:
I am using PyTorch and I want to change the learning rate after some epochs.
However, the code that is provided on most documentations, which is:
  optimizer = torch.optim.Adam([
        dict(params=model.parameters(), lr=learning_rate),
    ])
   #This line specifically
   optimizer.params_group[0]['lr'] = learning_rate

does not work.
Actually PyCharm hints at it:

Unresolved attribute reference 'params_group' for class 'Adam'

As a result, the error thrown is:

AttributeError: 'Adam' object has no attribute 'params_group'

How should one manually change the learning rate in PyTorch (1.6)?

Output:
Param_groups is not the feasible solution devised by the pytorch and thus you should be implementing pytorch.optim.lr_scheduler. Read more about this at other stackoverflow answer here.
import torch.optim.lr_scheduler.StepLR #step learning rate
scheduler = StepLR(optimizer, step_size=5, gamma=0.1)


Instruction: Access 3D tensor (image) using a 2d tensor of indices

Input:
With the following 3D tensor representing an image
img.shape=[H,W,F]
And a tensor representing the indices to that img
indices.shape=[N,2]
E.g. if indices = [[0,1],[5,3],...]]
I would like to create a new tensor of shape new.shape=[N,F] where new[k] == img[indices[k][0],indices[k][1]]
Currently to solve this I flatten both tensors:
    idx_flattened = idx_flattened [:,0] * (idx_flattened [:,1].max()+1) + idx_flattened[:,1]
    img = img .reshape(-1,F)
    new = img[idx_flattened ]

But I'm certain there is a better way:)
Here's a full minimal example:
img = torch.arange(8*10*3).reshape(8,10,3)
indices = torch.tensor([[0,0],[3,0],[1,2]])
new = img[indices] <- This does not work
new = [[  0,   1,   2],[ 90,  91,  92],[ 36,  37,  38]]

Ideas?

Output:
Slicing would work
img[indices[:,0], indices[:,1]]
tensor([[ 0,  1,  2],
        [90, 91, 92],
        [36, 37, 38]])


Instruction: how to see where exactly torch is installed pip vs conda torch installation

Input:
On my machine i can't "pip install torch" - i get infamous "single source externally managed error" - i could not fix it and used "conda install torch" from anaconda.
Still, checking version is easy - torch.__version__
But how to see where is it installed -the home dir of torch?
Suppose if I had had both torches installed via pip and conda - how to know which one is used in a project?
import torch
print(torch__version__)

Output:
You can get torch module location which is imported in your script
import torch
print(torch.__file__)


Instruction: Torchtext 0.7 shows Field is being deprecated. What is the alternative?

Input:
Looks like the previous paradigm of declaring Fields, Examples and using BucketIterator is deprecated and will move to legacy in 0.8. However, I don't seem to be able to find an example of the new paradigm for custom datasets (as in, not the ones included in torch.datasets) that doesn't use Field. Can anyone point me at an up-to-date example?
Reference for deprecation:
https://github.com/pytorch/text/releases

Output:
It took me a little while to find the solution myself.  The new paradigm is like so for prebuilt datasets:
from torchtext.experimental.datasets import AG_NEWS
train, test = AG_NEWS(ngrams=3)

or like so for custom built datasets:
from torch.utils.data import DataLoader
def collate_fn(batch):
    texts, labels = [], []
    for label, txt in batch:
        texts.append(txt)
        labels.append(label)
    return texts, labels
dataloader = DataLoader(train, batch_size=8, collate_fn=collate_fn)
for idx, (texts, labels) in enumerate(dataloader):
    print(idx, texts, labels)

I've copied the examples from the Source


Instruction: Why is the output of torch.lstsq drastically different than np.linalg.lstsq?

Input:
Pytorch provides a lstsq function, but the result it returns drastically differs from the numpy's version. Here is an example input and both of their results:
import numpy as np
import torch

a = torch.tensor([[1., 1, 1],
                  [2, 3, 4],
                  [3, 5, 2],
                  [4, 2, 5],
                  [5, 4, 3]])

b = torch.tensor([[-10., -3],
                  [ 12, 14],
                  [ 14, 12],
                  [ 16, 16],
                  [ 18, 16]])

a1 = a.clone().numpy()
b1 = b.clone().numpy()

x, r = torch.lstsq(b, a)

x1, res, r1, s = np.linalg.lstsq(b1, a1)

print(f'torch_x: {x}')
print(f'torch_r: {r}\n')

print(f'np_x: {x1}')
print(f'np_res: {res}')
print(f'np_r1(rank): {r1}')
print(f'np_s: {s}')

Output:
torch_x: tensor([[ 2.0000,  1.0000],
        [ 1.0000,  1.0000],
        [ 1.0000,  2.0000],
        [10.9635,  4.8501],
        [ 8.9332,  5.2418]])
torch_r: tensor([[-7.4162, -6.7420, -6.7420],
        [ 0.2376, -3.0896,  0.1471],
        [ 0.3565,  0.5272,  3.0861],
        [ 0.4753, -0.3952, -0.4312],
        [ 0.5941, -0.1411,  0.2681]])

np_x: [[-0.11452514 -0.10474861 -0.28631285]
 [ 0.35913807  0.33719075  0.54070234]]
np_res: [ 5.4269753 10.197526   1.4185953]
np_r1(rank): 2
np_s: [43.057705  5.199417]

What am I missing here?

Output:
torch.lstq(a, b) solves minX L2∥bX−a∥
while  np.linalg.lstsq(a, b) solves minX L2∥aX−b∥
So change the order of parameters passed.
Here's a sample:
import numpy as np
import torch
a = torch.tensor([[1., 1, 1],
                  [2, 3, 4],
                  [3, 5, 2],
                  [4, 2, 5],
                  [5, 4, 3]])

b = torch.tensor([[-10., -3],
                  [ 12, 14],
                  [ 14, 12],
                  [ 16, 16],
                  [ 18, 16]])

a1 = a.clone().numpy()
b1 = b.clone().numpy()

x, _ = torch.lstsq(a, b)

x1, res, r1, s = np.linalg.lstsq(b1, a1)

print(f'torch_x: {x[:b.shape[1]]}')

print(f'np_x: {x1}')

Results:
torch_x: tensor([[-0.1145, -0.1047, -0.2863],
        [ 0.3591,  0.3372,  0.5407]])
np_x: [[-0.11452514 -0.10474861 -0.28631285]
 [ 0.35913807  0.33719075  0.54070234]]

link to torch doc
link to numpy doc
And also the returned rank from numpy.lianalg.lstsq is rank of 1st parameters . To get rank in pytorch use torch.matrix_rank() function.


Instruction: In PyTorch is GPU memory freed when a GPU tensor is assigned a new value?

Input:
When a Cuda variable in PyTorch is assigned a new value, it becomes a CPU variable again (As illustrated by the code below). In this case, is the memory held by the variable on the GPU previously is freed automatically?
import torch

t1 = torch.empty(4,5)

if torch.cuda.is_available():
  t1 = t1.cuda()

print(t1.is_cuda)

t1 = torch.empty(4,5)
print(t1.is_cuda)

The output of the above code is:
True
False

Output:
In python an object is freed as soon as there are no remaining references to it. Since you assign t1 to reference a brand new tensor there are no more references to the original GPU tensor so that tensor is freed. That said, when PyTorch is instructed to free a GPU tensor it tends to cache that GPU memory for a while since it's usually the case that if we used GPU memory once we will probably want to use some again, and GPU memory allocation is relatively slow. If you want to force this cache of GPU memory to be cleared you can use torch.cuda.empty_cache. Using this won't directly increase the GPU memory available within a single PyTorch instance since PyTorch will call it automatically in an attempt to save you from an out of GPU memory error.
To reiterate, the GPU tensor doesn't actually "become" a CPU tensor. In python, variable names are references to objects. What your code really does is assign t1 to refer to a new CPU tensor object. Internally, python counts the number of references for each object. When that count goes to zero that object is immediately freed.
Caveat (Reference Cycles): Reference counting fails in the case of unreachable reference cycles. Unreachable reference cycles occur when objects contain references to one-another but no reference to any object in the cycle is reachable. To deal with this python employs a garbage collection module which executes intermittently. This module uses more sophisticated algorithms to detect and free objects that are part of unreachable reference cycles. In these cases, the memory is not necessarily freed when a cycle becomes unreachable and will instead be freed once the internal garbage collector is activated. This occurs automatically and relatively unpredictably. If desired, the garbage collector may be queried, configured, or manually executed using python's built in gc garbage collection interface.
Based on the previous discussion if you really want to ensure your unreachable GPU memory is freed in PyTorch (even in the case of unreachable reference cycles) you could use
import gc
gc.collect()
torch.cuda.empty_cache()


Instruction: How to extract feature vector from single image in Pytorch?

Input:
I am attempting to understand more about computer vision models, and I'm trying to do some exploring of how they work. In an attempt to understand how to interpret feature vectors more I'm trying to use Pytorch to extract a feature vector. Below is my code that I've pieced together from various places.
import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms
from torch.autograd import Variable
from PIL import Image

img=Image.open("Documents/01235.png")

# Load the pretrained model
model = models.resnet18(pretrained=True)

# Use the model object to select the desired layer
layer = model._modules.get('avgpool')

# Set model to evaluation mode
model.eval()

transforms = torchvision.transforms.Compose([
        torchvision.transforms.Resize(256),
        torchvision.transforms.CenterCrop(224),
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])

def get_vector(image_name):
    # Load the image with Pillow library
    img = Image.open("Documents/Documents/Driven Data Competitions/Hateful Memes Identification/data/01235.png")
    # Create a PyTorch Variable with the transformed image
    t_img = transforms(img)
    # Create a vector of zeros that will hold our feature vector
    # The 'avgpool' layer has an output size of 512
    my_embedding = torch.zeros(512)
    # Define a function that will copy the output of a layer
    def copy_data(m, i, o):
        my_embedding.copy_(o.data)
    # Attach that function to our selected layer
    h = layer.register_forward_hook(copy_data)
    # Run the model on our transformed image
    model(t_img)
    # Detach our copy function from the layer
    h.remove()
    # Return the feature vector
    return my_embedding

pic_vector = get_vector(img)

When I do this I get the following error:
RuntimeError: Expected 4-dimensional input for 4-dimensional weight [64, 3, 7, 7], but got 3-dimensional input of size [3, 224, 224] instead

I'm sure this is an elementary error, but I can't seem to figure out how to fix this. It was my impression that the "totensor" transformation would make my data 4-d, but it seems it's either not working correctly or I'm misunderstanding it. Appreciate any help or resources I can use to learn more about this!

Output:
All the default nn.Modules in pytorch expect an additional batch dimension. If the input to a module is shape (B, ...) then the output will be (B, ...) as well (though the later dimensions may change depending on the layer). This behavior allows efficient inference on batches of B inputs simultaneously. To make your code conform you can just unsqueeze an additional unitary dimension onto the front of t_img tensor before sending it into your model to make it a (1, ...) tensor. You will also need to flatten the output of layer before storing it if you want to copy it into your one-dimensional my_embedding tensor.
A couple of other things:

You should infer within a torch.no_grad() context to avoid computing gradients since you won't be needing them (note that model.eval() just changes the behavior of certain layers like dropout and batch normalization, it doesn't disable construction of the computation graph, but torch.no_grad() does).

I assume this is just a copy paste issue but transforms is the name of an imported module as well as a global variable.

o.data is just returning a copy of o. In the old Variable interface (circa PyTorch 0.3.1 and earlier) this used to be necessary, but the Variable interface was deprecated way back in PyTorch 0.4.0 and no longer does anything useful; now its use just creates  confusion. Unfortunately, many tutorials are still being written using this old and unnecessary interface.

Updated code is then as follows:
import torch
import torchvision
import torchvision.models as models
from PIL import Image

img = Image.open("Documents/01235.png")

# Load the pretrained model
model = models.resnet18(pretrained=True)

# Use the model object to select the desired layer
layer = model._modules.get('avgpool')

# Set model to evaluation mode
model.eval()

transforms = torchvision.transforms.Compose([
    torchvision.transforms.Resize(256),
    torchvision.transforms.CenterCrop(224),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

def get_vector(image):
    # Create a PyTorch tensor with the transformed image
    t_img = transforms(image)
    # Create a vector of zeros that will hold our feature vector
    # The 'avgpool' layer has an output size of 512
    my_embedding = torch.zeros(512)

    # Define a function that will copy the output of a layer
    def copy_data(m, i, o):
        my_embedding.copy_(o.flatten())                 # <-- flatten

    # Attach that function to our selected layer
    h = layer.register_forward_hook(copy_data)
    # Run the model on our transformed image
    with torch.no_grad():                               # <-- no_grad context
        model(t_img.unsqueeze(0))                       # <-- unsqueeze
    # Detach our copy function from the layer
    h.remove()
    # Return the feature vector
    return my_embedding

pic_vector = get_vector(img)


Instruction: Efficient way to compute Jacobian x Jacobian.T

Input:
Assume J is the Jacobian of some function f with respect to some parameters. Are there efficient ways (in PyTorch or perhaps Jax) to have a function that takes two inputs (x1 and x2) and computes J(x1)*J(x2).transpose() without instantiating the entire J matrices in memory?
I have come across something like jvp(f, input, v=vjp(f, input)) but don't quite understand it and not sure is what I want.

Output:
In JAX, you can compute a full jacobian matrix using jax.jacfwd or jax.jacrev, or you can compute a jacobian operator and its transpose using jax.jvp and jax.vjp.
So, for example, say you had a function Rᴺ → Rᴹ that looks something like this:
import jax.numpy as jnp
import numpy as np

np.random.seed(1701)
N, M = 10000, 5
f_mat = np.array(np.random.rand(M, N))
def f(x):
  return jnp.sqrt(f_mat @ x / N)

Given two vectors x1 and x2, you can evaluate the Jacobian matrix at each using jax.jacfwd
import jax
x1 = np.array(np.random.rand(N))
x2 = np.array(np.random.rand(N))
J1 = jax.jacfwd(f)(x1)
J2 = jax.jacfwd(f)(x2)
print(J1 @ J2.T)
# [[3.3123782e-05 2.5001222e-05 2.4946943e-05 2.5180108e-05 2.4940484e-05]
#  [2.5084497e-05 3.3233835e-05 2.4956826e-05 2.5108084e-05 2.5048916e-05]
#  [2.4969209e-05 2.4896170e-05 3.3232871e-05 2.5006309e-05 2.4947023e-05]
#  [2.5102483e-05 2.4947576e-05 2.4906987e-05 3.3327218e-05 2.4958186e-05]
#  [2.4981882e-05 2.5007204e-05 2.4966144e-05 2.5076926e-05 3.3595043e-05]]

But, as you note, along the way to computing this 5x5 result, we instantiate two 5x10,000 matrices. How might we get around this?
The answer is in jax.jvp and jax.vjp. These have somewhat unintuitive call signatures for the purposes of your question, as they are designed primarily for use in forward-mode and reverse-mode automatic differentiation. But broadly, you can think of them as a way to compute J @ v and J.T @ v for a vector v, without having to actually compute J explicitly.
For example, you can use jax.jvp to compute the effect of J1 operating on a vector, without actually computing J1:
J1_op = lambda v: jax.jvp(f, (x1,), (v,))[1]

vN = np.random.rand(N)
np.allclose(J1 @ vN, J1_op(vN))
# True

Similarly, you can use jax.vjp to compute the effect of J2.T operating on a vector, without actually computing J2:
J2T_op = lambda v: jax.vjp(f, x2)[1](v)[0]

vM = np.random.rand(M)
np.allclose(J2.T @ vM, J2T_op(vM))
# True

Putting these together and operating on an identity matrix gives you the full jacobian matrix product that you're after:
def direct(f, x1, x2):
  J1 = jax.jacfwd(f)(x1)
  J2 = jax.jacfwd(f)(x2)
  return J1 @ J2.T

def indirect(f, x1, x2, M):
  J1J2T_op = lambda v: jax.jvp(f, (x1,), jax.vjp(f, x2)[1](v))[1]
  return jax.vmap(J1J2T_op)(jnp.eye(M)).T

np.allclose(direct(f, x1, x2), indirect(f, x1, x2, M))
# True

Along with the memory savings, this indirect method is also a fair bit faster than the direct method, depending on the sizes of the jacobians involved:
%time direct(f, x1, x2)
# CPU times: user 1.43 s, sys: 14.9 ms, total: 1.44 s
# Wall time: 886 ms
%time indirect(f, x1, x2, M)
# CPU times: user 311 ms, sys: 0 ns, total: 311 ms
# Wall time: 158 ms


Instruction: Difference between model.train(False) and required_grad = False

Input:
I use the Pytorch library and i'm looking for a way to make the weights and biases in my model to freeze.
I saw these 2 options:

model.train(False)

for param in model.parameters(): param.requires_grad = False

What is the difference (if there is any) and which one should i use to freeze the current state of my model?

Output:
They are very different.
Independently of the backprop process, some layers have different behaviors when you are training or evaluating a model. In pytorch, there are only 2 of them : BatchNorm (which I think stops updating its running mean and deviation when you are evaluating) and Dropout (which only drops values in training mode). So model.train()and model.eval()(equivalently model.train(false)) just set a boolean flag to tell these 2 layers "freeze yourselves". Note that these two layers do not have any parameters that are affected by backward operation (batchnorm buffer tensors are changed during the forward pass I think)
On the other hand, setting all your parameters to "requires_grad=false" just tells pytorch to stop recording gradients for backprop. That will not affect the BatchNorm and the Dropout layers
How to freeze your model kinda depends on your use-case, but I'd say the easiest way is to use torch.jit.trace. This will create a frozen copy your model, exactly in the state it was when you called trace. Your model remained unaffected.
Usually, you would call
model.eval()
traced_model = torch.jit.trace(model, input)


Instruction: How to automatically disable register_hook when model is in eval() phase in PyTorch?

Input:
I require to update grads of an intermediate tensor variable using the register_hook method. Since the variable isn't a leaf-variable, I require to add the retain_grad() method to it after which, I can use the register_hook method to alter the grads.
score.retain_grad()
h = score.register_hook(lambda grad: grad * torch.FloatTensor(...))

This works perfectly fine during the training (model.train()) phase. However, it gives an error during the evaluation phase (model.eval()).
The error:
File "/home/envs/darthvader/lib/python3.6/site-packages/torch/tensor.py", line 198, in register_hook
    raise RuntimeError("cannot register a hook on a tensor that "
RuntimeError: cannot register a hook on a tensor that doesn't require gradient

How could the model automatically disable the register_hook method when it in eval() phase?

Output:
Removing score.retain_grad() and guarding register_hook with if condition (if score.requires_grad) does the trick.
if score.requires_grad:
    h = score.register_hook(lambda grad: grad * torch.FloatTensor(...))

Originally answered by Alban D here.


Instruction: RuntimeError: The size of tensor a (1024) must match the size of tensor b (512) at non-singleton dimension 3

Input:
I am doing the following operation,
energy.masked_fill(mask == 0, float("-1e20"))

my python traces are below,
    File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 418, in forward
    enc_src = self.encoder(src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 71, in forward
    src = layer(src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 110, in forward
    _src, _ = self.self_attention(src, src, src, src_mask)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "seq_sum.py", line 191, in forward
    energy =  energy.masked_fill(mask == 0, float("-1e20"))
RuntimeError: The size of tensor a (1024) must match the size of tensor b (512) at non-singleton dimension 3

These are my attention layers code,
    Q = self.fc_q(query)
    K = self.fc_k(key)
    V = self.fc_v(value)

    #Q = [batch size, query len, hid dim]
    #K = [batch size, key len, hid dim]
    #V = [batch size, value len, hid dim]

    # Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
    # K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
    # V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)

    Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    K = K.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    V = V.view(batch_size, -1, self.n_heads, self.head_dim).view(-1, 1024)
    energy = torch.matmul(Q, K.transpose(1,0)) / self.scale

I am following below github code to do my seq to seq operation,seq2seq pytorch
actual testing code is available on the below location, code to test a seq of 1024 to 1024 output
2nd example tried here I have commented out pos_embedding due CUDA error with large index (RuntimeError: cuda runtime error (59)

Output:
I took a look at your code (which by the way, didnt run with seq_len = 10) and the problem is that you hard coded the batch_size to be equal 1 (line 143) in your code.
It looks like the example you are trying to run the model on has batch_size = 2.
Just uncomment the previous line where you wrote batch_size = query.shape[0] and everything runs fine.


Instruction: Multiple-Input Linear Regression works for one dataset but not another

Input:
I am trying to run a logistic regression algorithm using Pytorch (and employing a neural network with one hidden layer), and I stumbled upon a problem. I am running the same algorithm for two different input data. My inputs are two-dimensional. The first set of data I created myself, while the second set of data come from real-world data which I got from a csv file, converted to lists and then to pytorch tensors.
For the first input data, the tensor I insert to the logistic regression code is:
 First tensor has rank
 torch.Size([1000, 2])

and it's given by:
T1= tensor([[ 0.6258,  0.9683],
        [-0.0833,  0.5691],
        [-0.4657, -0.8722],
        ...,
        [ 0.5868, -1.0565],
        [ 0.1611, -0.1716],
        [-0.1515, -0.8408]])

While the tensor for the second set of data is:
Second tensor has rank
 torch.Size([1064, 2])

and it's given by:
T2= tensor([[918.0600,  74.8220],
        [917.3477,  71.4038],
        [923.0400,  60.6380],
        ...,
        [916.6000,  71.0960],
        [912.6000,  58.4060],
        [921.5300,  77.7020]])

Now, for the first set of data, I get the following result:

So as you can see, the algorithm does a fairly good job with the reb/blue decision region, as most of the red points end up in the red region (and the same with the blue ones).
Now, for the second set of data, I get the following:

As you can see, it paints the whole region in red. I tried playing around with the number of neurons in my hidden layer and the learning rate, number of epochs and some other things, but nothing seems to work. I then thought that it might have to do with x-axis data having much larger values than the y-axis ones so I normalized them by dividing each with their mean, but this did not solve the problem.
The algorithm is the same, but it just doesn't work for this set of data. I was wondering if somebody who's more of an expert than me could have a hunch as to what might be going wrong here.

Output:
When you dont normalize the data the model can be easily fooled.
Your train set is composed of 1000 examples that by the looks of it, the majority of the values are in the range [-1, 1].
When you test your model however, you feed it with much much much higher numbers.
The solution is normalization. When you normalize your input your model can be free to learn the true distribution function of the data rather then "memorize" numbers.
You should normalize both the training set and the test set. Then your values will range between 0 and 1 and your network will have much better chance of picking up the desired correlation.
import torch
import torch.nn.functional as f

train = torch.rand((4, 2))*100

tensor([[36.9267,  7.3306],
        [63.5794, 42.9968],
        [61.3316, 67.6096],
        [88.4657, 11.7254]])

f.normalize(train, p=2, dim=1)

tensor([[0.9809, 0.1947],
        [0.8284, 0.5602],
        [0.6719, 0.7407],
        [0.9913, 0.1314]])


Instruction: Speeding up the trainning - RNN with LSTM in PyTorch

Input:
I am trying to train a LSTM for energy demand forecast but it takes too long. I do not understand why because the model looks “simple” and there is no much data. Might it be because I am not using the DataLoader? How could I use it with RNN since I have a sequence?
Complete code is in Colab: https://colab.research.google.com/drive/130rG8_j1Lf8RQoVRrfXCeo5h_CcC5NU6?usp=sharing
The interesting part to be improved may be this:
for seq, y_train in train_data:
    optimizer.zero_grad()
    model.hidden = (torch.zeros(1,1,model.hidden_size),
                    torch.zeros(1,1,model.hidden_size))
    y_pred = model(seq)
    loss = criterion(y_pred, y_train)
    loss.backward()
    optimizer.step()

Thanks in advance to anyone helping me.

Output:
Should you want to speed up the process of training, more data must be provided to the model per training. In my case I was providing just 1 batch. The best way to simply solve this is using the DataLoader.
Complete Colab with the solution can be found in this link: https://colab.research.google.com/drive/1QgtshCFETZ9oTvIYWy1Bdre-614kbwRX?usp=sharing
# This is to create the Dataset
from torch.utils.data import Dataset, DataLoader

class DemandDataset(Dataset):
    def __init__(self, X_train, y_train):
        self.X_train = X_train
        self.y_train = y_train

    def __len__(self):
        return len(self.y_train)

    def __getitem__(self, idx):
        data = self.X_train[idx]
        labels = self.y_train[idx]
        return data, labels

#This is to convert from typical RNN sequences
sq_0 =[]
y_0 =[]
for seq, y_train in train_data:
  sq_0.append(seq)
  y_0.append(y_train)

dataset=DemandDataset(sq_0,y_0)
dataloader = DataLoader(dataset, batch_size=20)

epochs = 30
t = 50

for i in range(epochs):
    print("New epoch")

    for data,label in dataloader:

        optimizer.zero_grad()
        model.hidden = (torch.zeros(1,1,model.hidden_size),
                        torch.zeros(1,1,model.hidden_size))

        y_pred = model(seq)

        loss = criterion(y_pred, label)
        loss.backward()
        optimizer.step()

    print(f'Epoch: {i+1:2} Loss: {loss.item():10.8f}')

    preds = train_set[-window_size:].tolist()

    for f in range(t):
        seq = torch.FloatTensor(preds[-window_size:])
        with torch.no_grad():
            model.hidden = (torch.zeros(1,1,model.hidden_size),
                            torch.zeros(1,1,model.hidden_size))
            preds.append(model(seq).item())

    loss = criterion(torch.tensor(preds[-window_size:]),y[-t:])


Instruction: How does Pytorch build the computation graph

Input:
Here is example pytorch code from the website:
class Net(nn.Module):

def __init__(self):
    super(Net, self).__init__()
    # 1 input image channel, 6 output channels, 3x3 square convolution
    # kernel
    self.conv1 = nn.Conv2d(1, 6, 3)
    self.conv2 = nn.Conv2d(6, 16, 3)
    # an affine operation: y = Wx + b
    self.fc1 = nn.Linear(16 * 6 * 6, 120)  # 6*6 from image dimension
    self.fc2 = nn.Linear(120, 84)
    self.fc3 = nn.Linear(84, 10)

def forward(self, x):
    # Max pooling over a (2, 2) window
    x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
    # If the size is a square you can only specify a single number
    x = F.max_pool2d(F.relu(self.conv2(x)), 2)
    x = x.view(-1, self.num_flat_features(x))
    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.fc3(x)
    return x

In the forward function, we simply apply a series of transformations to x, but never explicitly define which objects are part of that transformation. Yet when computing the gradient and updating the weights, Pytorch 'magically' knows which weights to update and how the gradient should be calculated.
How does this process work? Is there code analysis going on, or something else that I am missing?

Output:
Yes, there is implicit analysis on forward pass. Examine the result tensor, there is thingie like grad_fn= <CatBackward>, that's a link, allowing you to unroll the whole computation graph. And it is built during real forward computation process, no matter how you defined your network module, object oriented with 'nn' or 'functional' way.
You can exploit this graph for net analysis, as torchviz do here: https://github.com/szagoruyko/pytorchviz/blob/master/torchviz/dot.py


Instruction: type 'NoneType' is not iterable error when training pytorch model with ray tunes Trainable API

Input:
I wrote a simple pytorch script to train MNIST and it worked fine. I reimplemented my script to be with Trainable class:
import numpy as np
import torch
import torch.optim as optim
import torch.nn as nn
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F
import ray

from ray import tune

# Change these values if you want the training to run quicker or slower.
EPOCH_SIZE = 512
TEST_SIZE = 256

class ConvNet(nn.Module):

    def __init__(self):
        super(ConvNet, self).__init__()
        # In this example, we don't change the model architecture
        # due to simplicity.
        self.conv1 = nn.Conv2d(1, 3, kernel_size=3)
        self.fc = nn.Linear(192, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 3))
        x = x.view(-1, 192)
        x = self.fc(x)
        return F.log_softmax(x, dim=1)

class AlexTrainer(tune.Trainable):

    def setup(self, config):

        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        # Data Setup
        mnist_transforms = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])

        self.train_loader = DataLoader(
            datasets.MNIST("~/data", train=True, download=True, transform=mnist_transforms),
            batch_size=64,
            shuffle=True)
        self.test_loader = DataLoader(
            datasets.MNIST("~/data", train=False, transform=mnist_transforms),
            batch_size=64,
            shuffle=True)

        self.model = ConvNet()
        self.optimizer = optim.SGD(self.model.parameters(), lr=config["lr"], momentum=config["momentum"])

        print('finished setup')

    def step(self):

        self.train()
        print("after train")
        acc = self.test()

        return {'acc': acc}

    def train(self):

        print("in train")

        self.model.train()
        for batch_idx, (data, target) in enumerate(self.train_loader):

            # We set this just for the example to run quickly.
            if batch_idx * len(data) > EPOCH_SIZE:
                return

            data, target = data.to(self.device), target.to(self.device)
            self.optimizer.zero_grad()

            print(type(data))

            output = self.model(data)
            loss = F.nll_loss(output, target)
            loss.backward()

            self.optimizer.step()

    def test(self):
        self.model.eval()

        correct = 0
        total = 0
        with torch.no_grad():
            for batch_idx, (data, target) in enumerate(self.test_loader):
                # We set this just for the example to run quickly.
                if batch_idx * len(data) > TEST_SIZE:
                    break
                data, target = data.to(self.device), target.to(self.device)
                outputs = self.model(data)
                _, predicted = torch.max(outputs.data, 1)
                total += target.size(0)
                correct += (predicted == target).sum().item()

        return correct / total

if __name__ == '__main__':
    ray.init()
    analysis = tune.run(
        AlexTrainer,
        stop={"training_iteration": 2},
        # verbose=1,
        config={
            "lr": tune.sample_from(lambda spec: 10 ** (-10 * np.random.rand())),
            "momentum": tune.uniform(0.1, 0.9)
        }
    )

How ever when i try to run, this time it fails:
Traceback (most recent call last):
  File "/hdd/raytune/venv/lib/python3.6/site-packages/ray/tune/trial_runner.py", line 473, in _process_trial
    is_duplicate = RESULT_DUPLICATE in result
TypeError: argument of type 'NoneType' is not iterable
Traceback (most recent call last):
  File "/hdd/raytune/test_3.py", line 116, in <module>
    "momentum": tune.uniform(0.1, 0.9)
  File "/hdd/raytune/venv/lib/python3.6/site-packages/ray/tune/tune.py", line 356, in run
    raise TuneError("Trials did not complete", incomplete_trials)
ray.tune.error.TuneError: ('Trials did not complete', [AlexTrainer_9b3cd_00000])

What could be the reason for this?

Output: It's because you're actually overriding the existing train method in Trainable. If you rename your train method to something else it should work as expected.


Instruction: Why do we call .detach() before calling .numpy() on a Pytorch Tensor?

Input:
It has been firmly established that my_tensor.detach().numpy() is the correct way to get a numpy array from a torch tensor.
I'm trying to get a better understanding of why.
In the accepted answer to the question just linked, Blupon states that:

You need to convert your tensor to another tensor that isn't requiring a gradient in addition to its actual value definition.

In the first discussion he links to, albanD states:

This is expected behavior because moving to numpy will break the graph and so no gradient will be computed.
If you don’t actually need gradients, then you can explicitly .detach() the Tensor that requires grad to get a tensor with the same content that does not require grad. This other Tensor can then be converted to a numpy array.

In the second discussion he links to, apaszke writes:

Variable's can’t be transformed to numpy, because they’re wrappers around tensors that save the operation history, and numpy doesn’t have such objects. You can retrieve a tensor held by the Variable, using the .data attribute. Then, this should work: var.data.numpy().

I have studied the internal workings of PyTorch's autodifferentiation library, and I'm still confused by these answers.  Why does it break the graph to to move to numpy? Is it because any operations on the numpy array will not be tracked in the autodiff graph?
What is a Variable? How does it relate to a tensor?
I feel that a thorough high-quality Stack-Overflow answer that explains the reason for this to new users of PyTorch who don't yet understand autodifferentiation is called for here.
In particular, I think it would be helpful to illustrate the graph through a figure and show how the disconnection occurs in this example:

import torch

tensor1 = torch.tensor([1.0,2.0],requires_grad=True)

print(tensor1)
print(type(tensor1))

tensor1 = tensor1.numpy()

print(tensor1)
print(type(tensor1))

Output:
I think the most crucial point to understand here is the difference between a torch.tensor and np.ndarray:
While both objects are used to store n-dimensional matrices (aka "Tensors"), torch.tensors has an additional "layer" - which is storing the computational graph leading to the associated n-dimensional matrix.
So, if you are only interested in efficient and easy way to perform mathematical operations on matrices np.ndarray or torch.tensor can be used interchangeably.
However, torch.tensors are designed to be used in the context of gradient descent optimization, and therefore they hold not only a tensor with numeric values, but (and more importantly) the computational graph leading to these values. This computational graph is then used (using the chain rule of derivatives) to compute the derivative of the loss function w.r.t each of the independent variables used to compute the loss.
As mentioned before, np.ndarray object does not have this extra "computational graph" layer and therefore, when converting a torch.tensor to np.ndarray you must explicitly remove the computational graph of the tensor using the detach() command.

Computational Graph
From your comments it seems like this concept is a bit vague. I'll try and illustrate it with a simple example.
Consider a simple function of two (vector) variables, x and w:
x = torch.rand(4, requires_grad=True)
w = torch.rand(4, requires_grad=True)

y = x @ w  # inner-product of x and w
z = y ** 2  # square the inner product

If we are only interested in the value of z, we need not worry about any graphs, we simply moving forward from the inputs, x and w, to compute y and then z.
However, what would happen if we do not care so much about the value of z, but rather want to ask the question "what is w that minimizes z for a given x"?
To answer that question, we need to compute the derivative of z w.r.t w.
How can we do that?
Using the chain rule we know that dz/dw = dz/dy * dy/dw. That is, to compute the gradient of z w.r.t w we need to move backward from z back to w computing the gradient of the operation at each step as we trace back our steps from z to w. This "path" we trace back is the computational graph of z and it tells us how to compute the derivative of z w.r.t the inputs leading to z:
z.backward()  # ask pytorch to trace back the computation of z

We can now inspect the gradient of z w.r.t w:

w.grad  # the resulting gradient of z w.r.t w
tensor([0.8010, 1.9746, 1.5904, 1.0408])

Note that this is exactly equals to

2*y*x
tensor([0.8010, 1.9746, 1.5904, 1.0408], grad_fn=<MulBackward0>)

since dz/dy = 2*y and dy/dw = x.
Each tensor along the path stores its "contribution" to the computation:

z
tensor(1.4061, grad_fn=<PowBackward0>)

And

y
tensor(1.1858, grad_fn=<DotBackward>)

As you can see, y and z stores not only the "forward" value of <x, w> or y**2 but also the computational graph -- the grad_fn that is needed to compute the derivatives (using the chain rule) when tracing back the gradients from z (output) to w (inputs).
These grad_fn are essential components to torch.tensors and without them one cannot compute derivatives of complicated functions. However, np.ndarrays do not have this capability at all and they do not have this information.
please see this answer for more information on tracing back the derivative using backwrd() function.

Since both np.ndarray and torch.tensor has a common "layer" storing an n-d array of numbers, pytorch uses the same storage to save memory:

numpy() → numpy.ndarray
Returns self tensor as a NumPy ndarray. This tensor and the returned ndarray share the same underlying storage. Changes to self tensor will be reflected in the ndarray and vice versa.

The other direction works in the same way as well:

torch.from_numpy(ndarray) → Tensor
Creates a Tensor from a numpy.ndarray.
The returned tensor and ndarray share the same memory. Modifications to the tensor will be reflected in the ndarray and vice versa.

Thus, when creating an np.array from torch.tensor or vice versa, both object reference the same underlying storage in memory. Since np.ndarray does not store/represent the computational graph associated with the array, this graph should be explicitly removed using detach() when sharing both numpy and torch wish to reference the same tensor.

Note, that if you wish, for some reason, to use pytorch only for mathematical operations without back-propagation, you can use with torch.no_grad() context manager, in which case computational graphs are not created and torch.tensors and np.ndarrays can be used interchangeably.
with torch.no_grad():
  x_t = torch.rand(3,4)
  y_np = np.ones((4, 2), dtype=np.float32)
  x_t @ torch.from_numpy(y_np)  # dot product in torch
  np.dot(x_t.numpy(), y_np)  # the same dot product in numpy


Instruction: What is the Pytorch sub for this tensor flow code?

Input:
In converting this line of code to Pytorch from Tensor Flow, I am having trouble
datagen =  ImageDataGenerator(
  shear_range=0.2,
  zoom_range=0.2,
)
def read_img(filename, size, path):
    img = image.load_img(os.path.join(path, filename), target_size=size)
    #convert image to array
    img = img_to_array(img) / 255
    return img

and then
corona_df = final_train_data[final_train_data['Label_2_Virus_category'] == 'COVID-19']
with_corona_augmented = []

#create a function for augmentation
def augment(name):
    img = read_img(name, (255,255), train_img_dir)
    i = 0
    for batch in tqdm(datagen.flow(tf.expand_dims(img, 0), batch_size=32)):
        with_corona_augmented.append(tf.squeeze(batch).numpy())
        if i == 20:
            break
        i =i+1

#apply the function
corona_df['X_ray_image_name'].apply(augment)

I tried doing
transform = transforms.Compose([transforms.Resize(255*255)
                               ])
train_loader = torch.utils.data.DataLoader(os.path.join(train_dir,corona_df),transform = transform,batch_size =32)
def read_img(path):
    img  = train_loader()
    img = np.asarray(img,dtype='int32')
    img = img/255
    return img

I tried continuing but got soo confused by the errors.
I welcome any feedback. Tell me If i miss something
Even a small advice would work, thanks !

Output:
You can create a custom dataset to read the images. If you have a directory full of images you can use ImageFolder default dataset. Otherwise if you have different folder placement you can write your own custom dataset class. You can look to this link for custom datasets. What dataloader does is, it automatically gets the data from your dataset and read the images according to your dataset __getitem__ function and apply transformation. So you don't need anything fancy to apply augmentation.
transform = transforms.Compose([ transforms.RandomAffine(20,shear=20,scale=(-0.2,0.2)),
                                 transforms.Resize(255*255)
                               ])

dataset = torchvision.datasets.ImageFolder(train_img_dir, transform=transform)
loader = torch.utils.data.DataLoader(dataset,batch_size =32,shuffle=True)

for batch in loader:
    output = model(batch)


Instruction: Feature extraction in loop seems to cause memory leak in pytorch

Input:
I have spent considerable time trying to debug some pytorch code which I have created a minimal example of for the purpose of helping to better understand what the issue might be.
I have removed all necessary portions of the code which are unrelated to the issue so the remaining piece of code won't make much sense from a functional standpoint but it still displays the error I'm facing.
The overall task I'm working on is in a loop and every pass of the loop is computing the embedding of the image and adding it to a variable storing it. It's effectively aggregating it (not concatenating, so the size remains the same). I don't expect the number of iterations to force the datatype to overflow, I don't see this happening here nor in my code.

I have added multiple metrics to evaluate the size of the tensors I'm working with to make sure they're not growing in memory footprint
I'm checking the overall GPU memory usage to verify the issue leading to the final RuntimeError: CUDA out of memory..

My environment is as follows:
 - python 3.6.2
 - Pytorch 1.4.0
 - Cudatoolkit 10.0
 - Driver version 410.78
 - GPU: Nvidia GeForce GT 1030  (2GB VRAM)
(though I've replicated this experiment with the same result on a Titan RTX with 24GB,
same pytorch version and cuda toolkit and driver, it only goes out of memory further in the loop).

Complete code below. I have marked 2 lines as culprits, as deleting them removes the issue, though obviously I need to find a way to execute them without having memory issues. Any help would be much appreciated! You may try with any image named "source_image.bmp" to replicate the issue.
import torch
from PIL import Image
import torchvision
from torchvision import transforms
from pynvml import nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo, nvmlInit
import sys
import os
os.environ["CUDA_VISIBLE_DEVICES"]='0'      # this is necessary on my system to allow the environment to recognize my nvidia GPU for some reason
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'    # to debug by having all CUDA functions executed in place
torch.set_default_tensor_type('torch.cuda.FloatTensor')

# Preprocess image
tfms = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),])
img = tfms(Image.open('source_image.bmp')).unsqueeze(0).cuda()

model = torchvision.models.resnet50(pretrained=True).cuda()
model.eval()    # we put the model in evaluation mode, to prevent storage of gradient which might accumulate

nvmlInit()
h = nvmlDeviceGetHandleByIndex(0)
info = nvmlDeviceGetMemoryInfo(h)
print(f'Total available memory   : {info.total / 1000000000}')

feature_extractor = torch.nn.Sequential(*list(model.children())[:-1])
orig_embedding = feature_extractor(img)

embedding_depth = 2048

mem0 = 0

embedding = torch.zeros(2048, img.shape[2], img.shape[3]) #, dtype=torch.float)

patch_size=[4,4]
patch_stride=[2,2]
patch_value=0.0

# Here, we iterate over the patch placement, defined at the top left location
for row in range(img.shape[2]-1):
    for col in range(img.shape[3]-1):
        print("######################################################")

        ######################################################
        # Isolated line, culprit 1 of the GPU memory leak
        ######################################################
        patched_embedding = feature_extractor(img)

        delta_embedding = (patched_embedding - orig_embedding).view(-1, 1, 1)

        ######################################################
        # Isolated line, culprit 2 of the GPU memory leak
        ######################################################
        embedding[:,row:row+1,col:col+1] = torch.add(embedding[:,row:row+1,col:col+1], delta_embedding)

        print("img size:\t\t", img.element_size() * img.nelement())
        print("patched_embedding size:\t", patched_embedding.element_size() * patched_embedding.nelement())
        print("delta_embedding size:\t", delta_embedding.element_size() * delta_embedding.nelement())
        print("Embedding size:\t\t", embedding.element_size() * embedding.nelement())

        del patched_embedding, delta_embedding
        torch.cuda.empty_cache()

        info = nvmlDeviceGetMemoryInfo(h)
        print("\nMem usage increase:\t", info.used / 1000000000 - mem0)
        mem0 = info.used / 1000000000
        print(f'Free:\t\t\t {(info.total - info.used) / 1000000000}')

print("Done.")

Output:
Add this to your code as soon as you load the model
for param in model.parameters():
    param.requires_grad = False

from https://pytorch.org/docs/stable/notes/autograd.html#excluding-subgraphs-from-backward


Instruction: PyTorch: How to do inference in batches (inference in parallel)

Input:
How to do inference in batches in PyTorch? How to do inference in parallel to speed up that part of the code.
I've started with the standard way of doing inference:
with torch.no_grad():
    for inputs, labels in dataloader['predict']:
        inputs = inputs.to(device)
        output = model(inputs)
        output = output.to(device)

And I've researched and the only mention of doing inference in parallel (in the same machine) seems to be with the library Dask: https://examples.dask.org/machine-learning/torch-prediction.html
Currently attempting to understand that library and create a working example. In the meanwhile do you know of a better way?

Output:
In pytorch, the input tensors always have the batch dimension in the first dimension. Thus doing inference by batch is the default behavior, you just need to increase the batch dimension to larger than 1.
For example, if your single input is [1, 1], its input tensor is [[1, 1], ] with shape (1, 2). If you have two inputs [1, 1] and [2, 2], generate the input tensor as [[1, 1], [2, 2], ] with shape (2, 2). This is usually done in the batch generator function such as your dataloader.


Instruction: Torch Tensor & Input Conflicting: "Tensor Object Is Not Callable"

Input:
Due to the code "torch.tensor," I am getting the error "Tensor object is not callable" when I add "input." Does anyone know how I can fix this?
import torch
from torch.nn import functional as F
from transformers import GPT2Tokenizer, GPT2LMHeadModel

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

text0 = "In order to"
text = tokenizer.encode("In order to")
input, past = torch.tensor([text]), None

logits, past = model(input, past = past)
logits = logits[0,-1]
probabilities = torch.nn.functional.softmax(logits)
best_logits, best_indices = logits.topk(5)
best_words = [tokenizer.decode([idx.item()]) for idx in best_indices]
text.append(best_indices[0].item())
best_probabilities = probabilities[best_indices].tolist()

for i in range(5):
    f = ('Generated {}: {}'.format(i, best_words[i]))
    print(f)

option = input("Pick a Option:")
z = text0.append(option)
print(z)

Error stacktrace:
TypeError                                 Traceback (most recent call last)

<ipython-input-2-82e8d88e81c1> in <module>()
     25
     26
---> 27 option = input("Pick a Option:")
     28 z = text0.append(option)
     29 print(z)

TypeError: 'Tensor' object is not callable

Output:
The problem is that you have already defined a variable with the name input which will be used instead of the input function. Just use a different name for your variable and it will work as expected.
Also a python string does not have an append method.
import torch
from torch.nn import functional as F
from transformers import GPT2Tokenizer, GPT2LMHeadModel

tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

text0 = "In order to"
text = tokenizer.encode("In order to")
myinput, past = torch.tensor([text]), None

logits, past = model(myinput, past = past)
logits = logits[0,-1]
probabilities = torch.nn.functional.softmax(logits)
best_logits, best_indices = logits.topk(5)
best_words = [tokenizer.decode([idx.item()]) for idx in best_indices]
text.append(best_indices[0].item())
best_probabilities = probabilities[best_indices].tolist()

for i in range(5):
    f = ('Generated {}: {}'.format(i, best_words[i]))
    print(f)

option = input("Pick a Option:")
z = text0 + ' ' + option
print(z)


Instruction: Preprocessing image for Tensorflow model instead Pytorch preprocessing

Input:
I had a pytorch resnet101 encoder model, when input image got this preprocessing:
import torchvision as tv
from PIL import Image

data_transforms = tv.transforms.Compose([
    tv.transforms.Resize((224, 224)),
    tv.transforms.ToTensor(),
    tv.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
img = Image.open(img_path)
img = img.convert('RGB')
img = data_transforms(img)
img = torch.FloatTensor(img)
img = img.unsqueeze(0)
print(img)

pytorch image tensor
Shape of input for encoder in this case is [1, 3, 224, 224], and this picture is normalized with the mean and std of ImageNet.
Now I'm export this model to tensorflow, so, how to make the same image preprocessing for tf-model?
I tried to do something like this:
from PIL import Image

img = Image.open(img_path)
img = img.convert('RGB')
img = tf.keras.preprocessing.image.img_to_array(img)
img = tf.image.resize(tf_img, (224, 224))
img = tf.keras.applications.resnet.preprocess_input(img)# now shape is [224, 224, 3]
img = tf.reshape(img, [1, 3, 224, 224])
print(img)

tensorflow image tensor
but I'm sure that I did something wrong, becouse torch and tf tensors looks very different for one image and give completely different output results from one encoder model.
Can anyone helps, what should I fix in tf preprocessing?

Output:
This:
img = Image.open(img_path)
img = img.convert('RGB')

could be replaced with
image = tf.io.read_file(filename=filepath)
image = tf.image.decode_jpeg(image, channels=3) #or decode_png

Also, the opposite of unsqueeze and squeeze is expand_dims:
  img = tf.expand_dims(img,axis=0)

Everything should work well, just ensure that
tf.keras.applications.resnet.preprocess_input(img) `and` data.transforms()

yield the desired/necessary transformations.
As for the photos, I am quite sure that you missed a /255.0 in case of PyTorch or added a 255.0 division in case of TensorFlow.
In fact, when digging deep into the Keras backend, you can see that when you call your preprocessing function, it will call this function here:
def _preprocess_numpy_input(x, data_format, mode):
  """Preprocesses a Numpy array encoding a batch of images.

  Arguments:
    x: Input array, 3D or 4D.
    data_format: Data format of the image array.
    mode: One of "caffe", "tf" or "torch".
      - caffe: will convert the images from RGB to BGR,
          then will zero-center each color channel with
          respect to the ImageNet dataset,
          without scaling.
      - tf: will scale pixels between -1 and 1,
          sample-wise.
      - torch: will scale pixels between 0 and 1 and then
          will normalize each channel with respect to the
          ImageNet dataset.

  Returns:
      Preprocessed Numpy array.
  """
  if not issubclass(x.dtype.type, np.floating):
    x = x.astype(backend.floatx(), copy=False)

  if mode == 'tf':
    x /= 127.5
    x -= 1.
    return x
  elif mode == 'torch':
    x /= 255.
    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]
  else:
    if data_format == 'channels_first':
      # 'RGB'->'BGR'
      if x.ndim == 3:
        x = x[::-1, ...]
      else:
        x = x[:, ::-1, ...]
    else:
      # 'RGB'->'BGR'
      x = x[..., ::-1]
    mean = [103.939, 116.779, 123.68]
    std = None

  # Zero-center by mean pixel
  if data_format == 'channels_first':
    if x.ndim == 3:
      x[0, :, :] -= mean[0]
      x[1, :, :] -= mean[1]
      x[2, :, :] -= mean[2]
      if std is not None:
        x[0, :, :] /= std[0]
        x[1, :, :] /= std[1]
        x[2, :, :] /= std[2]
    else:
      x[:, 0, :, :] -= mean[0]
      x[:, 1, :, :] -= mean[1]
      x[:, 2, :, :] -= mean[2]
      if std is not None:
        x[:, 0, :, :] /= std[0]
        x[:, 1, :, :] /= std[1]
        x[:, 2, :, :] /= std[2]
  else:
    x[..., 0] -= mean[0]
    x[..., 1] -= mean[1]
    x[..., 2] -= mean[2]
    if std is not None:
      x[..., 0] /= std[0]
      x[..., 1] /= std[1]
      x[..., 2] /= std[2]
  return x

The default mode parameter in Keras and TensorFlow for preprocessing for ResNet50, is surprisingly not tf but caffe.
Therefore, the preprocessing that is done to the image is on this else branch( I am adding the else branch and the code thereafter so that you can follow the transformations and see what you are missing):
 else:
    if data_format == 'channels_first':
      # 'RGB'->'BGR'
      if x.ndim == 3:
        x = x[::-1, ...]
      else:
        x = x[:, ::-1, ...]
    else:
      # 'RGB'->'BGR'
      x = x[..., ::-1]
    mean = [103.939, 116.779, 123.68]
    std = None
# Zero-center by mean pixel
  if data_format == 'channels_first':
    if x.ndim == 3:
      x[0, :, :] -= mean[0]
      x[1, :, :] -= mean[1]
      x[2, :, :] -= mean[2]
      if std is not None:
        x[0, :, :] /= std[0]
        x[1, :, :] /= std[1]
        x[2, :, :] /= std[2]
    else:
      x[:, 0, :, :] -= mean[0]
      x[:, 1, :, :] -= mean[1]
      x[:, 2, :, :] -= mean[2]
      if std is not None:
        x[:, 0, :, :] /= std[0]
        x[:, 1, :, :] /= std[1]
        x[:, 2, :, :] /= std[2]
  else:
    x[..., 0] -= mean[0]
    x[..., 1] -= mean[1]
    x[..., 2] -= mean[2]
    if std is not None:
      x[..., 0] /= std[0]
      x[..., 1] /= std[1]
      x[..., 2] /= std[2]
  return x

The description is:

caffe: will convert the images from RGB to BGR,
then will zero-center each color channel with
respect to the ImageNet dataset,
without scaling.


Instruction: Pytorch geometric: Having issues with tensor sizes

Input:
This is the first time I'm using Pytorch and Pytorch geometric. I'm trying to create a simple Graph Neural Network with Pytorch Geometric. I'm creating a custom dataset by following the Pytorch Geometric documentations and extending the InMemoryDataset. After that I split the dataset into training, validation and test dataset which sizes (3496, 437, 439) respectively. These are the number of graphs in each dataset. Here is my simple Neural Network
class Net(torch.nn.Module):
def __init__(self):
    super(Net, self).__init__()
    self.conv1 = GCNConv(dataset.num_node_features, 10)
    self.conv2 = GCNConv(10, dataset.num_classes)

def forward(self, data):
    x, edge_index, batch = data.x, data.edge_index, data.batch
    x = self.conv1(x, edge_index)
    x = F.relu(x)
    x = F.dropout(x, training=self.training)
    x = self.conv2(x, edge_index)

    return F.log_softmax(x, dim=1)

I get this error while training my model, which suggest that there's some issue with my input dimensions. Maybe the reason is behind my batch sizes?
RuntimeError: The following operation failed in the TorchScript interpreter.
Traceback of TorchScript (most recent call last):
File "E:\Users\abc\Anaconda3\lib\site-packages\torch_scatter\scatter.py", line 22, in scatter_add
        size[dim] = int(index.max()) + 1
    out = torch.zeros(size, dtype=src.dtype, device=src.device)
    return out.scatter_add_(dim, index, src)
           ~~~~~~~~~~~~~~~~ <--- HERE
else:
    return out.scatter_add_(dim, index, src)
RuntimeError: index 13654 is out of bounds for dimension 0 with size 678

The error happens specifically on this line of code in the Neural Network,
x = self.conv1(x, edge_index)

EDIT: Added more information about edge_index and explained in more detail about the data that I'm using.
Here are the shapes of the variables that I'm trying to pass
x: torch.Size([678, 43])
edge_index: torch.Size([2, 668])
torch.max(edge_index): tensor(541690)
torch.min(edge_index): tensor(1920)

I'm using a datalist which contains Data(x=node_features, edge_index=edge_index, y=labels) objects. When I'm splitting the dataset into training, validation and test datasets, I get (3496, 437, 439) graphs in each dataset respectively. Originally I tried to create one single graph from my dataset, but I'm not sure how it would work with Dataloader and minibatches.
train_loader = DataLoader(train_dataset, batch_size=batch_size)
val_loader = DataLoader(val_dataset, batch_size=batch_size)
test_loader = DataLoader(test_dataset, batch_size=batch_size)

Here's the code that generates the graph from dataframe. I've tried to create an simple graph where there are just some amount of vertices with some amount of edges connecting them. I've probably overlooked something and that's why I have this issue. I've tried to follow the Pytorch geometric documentation when creating this graph (Pytorch Geometric: Creating your own dataset)
def process(self):
        data_list = []

        grouped = df.groupby('EntityId')
        for id, group in grouped:
            node_features = torch.tensor(group.drop(['Labels'], axis=1).values)
            source_nodes = group.index[1:].values
            target_nodes = group.index[:-1].values
            labels = torch.tensor(group.Labels.values)
            edge_index = torch.tensor([source_nodes, target_nodes])

            data = Data(x=node_features, edge_index=edge_index, y=labels)
            data_list.append(data)

        if self.pre_filter is not None:
            data_list = [data for data in data_list if self.pre_filter(data)]

        if self.pre_transform is not None:
            data_list = [self.pre_transform(data) for data in data_list]

        data, slices = self.collate(data_list)
        torch.save((data, slices), self.processed_paths[0])

If someone could help me with the process of creating a graph on any kind of data and using it with GCNConv, I would appreciate it.

Output:
I agree with @trialNerror -- it is a data problem. Your edge_index should refer to the data nodes and its max should not be that high. Since you don't want to show us the data and ask for "creating a graph on any kind of data ", here it is.
I mostly left your Net unchanged. You can play around with the constants stated to match with your data.
import torch
import torch.nn as nn
import torch.nn.functional as F

from torch_geometric.nn import GCNConv
from torch_geometric.data import Data

num_node_features = 100
num_classes = 2
num_nodes = 678
num_edges = 1500
num_hidden_nodes = 128

x = torch.randn((num_nodes, num_node_features), dtype=torch.float32)
edge_index = torch.randint(low=0, high=num_nodes, size=(2, num_edges), dtype=torch.long)
y = torch.randint(low=0, high=num_classes, size=(num_nodes,), dtype=torch.long)

class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = GCNConv(num_node_features, num_hidden_nodes)
        self.conv2 = GCNConv(num_hidden_nodes, num_classes)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

data = Data(x=x, edge_index=edge_index, y=y)

net = Net()

optimizer = torch.optim.Adam(net.parameters(), lr=1e-2)
for i in range(1000):
    output = net(data)
    loss = F.cross_entropy(output, data.y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    if i % 100 == 0:
        print('Accuracy: ', (torch.argmax(output, dim=1)==data.y).float().mean())

Output
Accuracy:  tensor(0.5059)
Accuracy:  tensor(0.8702)
Accuracy:  tensor(0.9159)
Accuracy:  tensor(0.9233)
Accuracy:  tensor(0.9336)
Accuracy:  tensor(0.9484)
Accuracy:  tensor(0.9602)
Accuracy:  tensor(0.9676)
Accuracy:  tensor(0.9705)
Accuracy:  tensor(0.9749)

(yes we can overfit to random data)