Instruction: Value Error: Torch target size and torch input size in GAN do not match

Input:
Hi I  am working on a GAN with custom images. I got the following error, which doesn't add up for me:
ValueError: Using a target size (torch.Size([64, 1])) that is different to the input size (torch.Size([47, 1])) is deprecated. Please ensure they have the same size.
I do not see where either of these sizes come from. Could someone please help me out? The error is to be found at the loss_disrimenator in the course of the training (marked with an arrow) after epoch 0. Below you find the related code. I am using vs code windows.
Also is it normal that epoch 0 works and then the problem appears?
[Sceenshot of Terminal- Epoch 0 Loss Discriminatorand Generator][1]
import torch
from glob import glob
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from skimage import io
import matplotlib.pyplot as plt

path = 'Punks'
image_paths = glob(path + '/*.png')

img_size = 28
batch_size = 32

transform = transforms.Compose(
    [
        transforms.ToPILImage(),
        transforms.Resize(img_size),
        transforms.CenterCrop(img_size),
        transforms.ToTensor(),
        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
    ]
)

class ImageDataset(Dataset):
    def __init__(self, paths, transform):
        self.paths = paths
        self.transform = transform

    def __len__(self):
        return len(self.paths)

    def __getitem__(self, index):
        image_path = self.paths[index]
        image = io.imread(image_path)

        if self.transform:
            image_tensor = self.transform(image)

        return image_tensor

if __name__ == '__main__':

    dataset = ImageDataset(image_paths, transform)

    train_loader = DataLoader(
        dataset, batch_size=batch_size, num_workers=1, shuffle=True)

    # PLOTTING SAMPLES

    real_samples = next(iter(train_loader))
    for i in range(9):
        ax = plt.subplot(3, 3, 3 + 1)
        plt.imshow(real_samples[i].reshape(28, 28, 3))
        plt.xticks([])
        plt.yticks([])
        plt.show()

    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    class Discriminator(nn.Module):
        def __init__(self):
            super().__init__()
            self.model = nn.Sequential(
                nn.Linear(784*3, 2048),
                nn.ReLU(),
                nn.Dropout(0.3),
                nn.Linear(2048, 1024),
                nn.ReLU(),
                nn.Dropout(0.3),
                nn.Linear(1024, 512),
                nn.ReLU(),
                nn.Dropout(0.3),
                nn.Linear(512, 256),
                nn.ReLU(),
                nn.Dropout(0.3),
                nn.Linear(256, 1),
                nn.Sigmoid(),
            )

        def forward(self, x):
            x = x.view(x.size(0), 784*3)  # change required for 3 channel image
            output = self.model(x)
            return output

    discriminator = Discriminator().to(device=device)

    class Generator(nn.Module):
        def __init__(self):
            super().__init__()
            self.model = nn.Sequential(
                nn.Linear(100, 256),
                nn.ReLU(),
                nn.Linear(256, 512),
                nn.ReLU(),
                nn.Linear(512, 1024),
                nn.ReLU(),
                nn.Linear(1024, 2048),
                nn.ReLU(),
                nn.Linear(2048, 784*3),
                nn.Tanh(),
            )

        def forward(self, x):
            output = self.model(x)
            output = output.view(x.size(0), 3, 28, 28)
            return output

    generator = Generator().to(device=device)

    # TRAINING PARAMS

    lr = 0.0001
    num_epochs = 10
    loss_function = nn.BCELoss()

    optimizer_discriminator = torch.optim.Adam(discriminator.parameters(), lr=lr)
    optimizer_generator = torch.optim.Adam(generator.parameters(), lr=lr)
    for epoch in range(num_epochs):
        for n, real_samples in enumerate(train_loader):
            # Data for training the discriminator
            real_samples = real_samples.to(device=device)
            real_samples_labels = torch.ones((batch_size, 1)).to(
                device=device
            )
            latent_space_samples = torch.randn((batch_size, 100)).to(
                device=device
            )
            print(f'Latent space samples : {latent_space_samples.shape}')
            generated_samples = generator(latent_space_samples)
            generated_samples_labels = torch.zeros((batch_size, 1)).to(
                device=device
            )
            all_samples = torch.cat((real_samples, generated_samples))
            print(f'Real samples : {real_samples.shape}, generated samples : {generated_samples.shape}')
            all_samples_labels = torch.cat(
                (real_samples_labels, generated_samples_labels)
            )

            # Training the discriminator
            discriminator.zero_grad()
            output_discriminator = discriminator(all_samples)
            loss_discriminator = loss_function(
                output_discriminator, all_samples_labels
            )
 ------->    loss_discriminator.backward()
            optimizer_discriminator.step()

            # Data for training the generator
            latent_space_samples = torch.randn((batch_size, 100)).to(
                device=device
            )

            # Training the generator
            generator.zero_grad()
            generated_samples = generator(latent_space_samples)
            output_discriminator_generated = discriminator(generated_samples)
            loss_generator = loss_function(
                output_discriminator_generated, real_samples_labels
            )
            loss_generator.backward()
            optimizer_generator.step()

            # Show loss
            if n == batch_size - 1:
                print(f"Epoch: {epoch} Loss D.: {loss_discriminator}")
                print(f"Epoch: {epoch} Loss G.: {loss_generator}")

    latent_space_samples = torch.randn(batch_size, 100).to(device=device)
    generated_samples = generator(latent_space_samples)

    generated_samples = generated_samples.cpu().detach()
    for i in range(9):
        ax = plt.subplot(3, 3, i + 1)
        plt.imshow(generated_samples[i].reshape(28, 28, 3))
        plt.xticks([])
        plt.yticks([])
        plt.show()´´´

Output:
Since this is happening at the end of the first epoch, what's essentially happening is that you have specified a batch size of 64 but the number of images in your dataset is some_integer_number * 64 + 47. This is because when you read the data in batches, the number of samples equal to your batch_size is read. However, when you reach the end of the epoch, there is a possibility that fewer than batch_size examples are left to load.
In your code, the number of generated images in the last step of the 0th epoch is 47 whereas the number of fake images that you are generating is 64 since you use batch_size to sample batch_size number of fake images.
A simple solution would be to use len(real_samples) in place of batch_size at all the places. You can do this by first setting batch_size=len(real_samples) as the first line in the for loop.
for epoch in range(num_epochs):
   for n, real_samples in enumerate(train_loader):
      # Data for training the discriminator
      batch_size = len(real_samples)
      real_samples = real_samples.to(device=device)
      real_samples_labels = torch.ones((batch_size, 1)).to(device=device)

      # rest of the code continues

I hope this solves your issue.


Instruction: pytorch.info summary for Generator of AN equivalent to discriminator or summary for whole GAN

Input:
Is it possible to generate a summary for the generator network of a GAN equivalent to the summary for the discriminator network using pytorch.info (containing inputs and outputs) or is there even a standard summary for the whole GAN including both networks?
For the Discriminator I used the following:
model = Discriminator()
batch_size = 32
summary(model, input_size=(batch_size, 3, 28, 28))

and received the following summary, which I would also like for the generator (see below summary):
==========================================================================================
Layer (type:depth-idx)                   Output Shape              Param #
==========================================================================================
Discriminator                            [32, 1]                   --
├─Sequential: 1-1                        [32, 1]                   --
│    └─Linear: 2-1                       [32, 2048]                4,818,944
│    └─ReLU: 2-2                         [32, 2048]                --
│    └─Dropout: 2-3                      [32, 2048]                --
│    └─Linear: 2-4                       [32, 1024]                2,098,176
│    └─ReLU: 2-5                         [32, 1024]                --
│    └─Dropout: 2-6                      [32, 1024]                --
│    └─Linear: 2-7                       [32, 512]                 524,800
│    └─ReLU: 2-8                         [32, 512]                 --
│    └─Dropout: 2-9                      [32, 512]                 --
│    └─Linear: 2-10                      [32, 256]                 131,328
│    └─ReLU: 2-11                        [32, 256]                 --
│    └─Dropout: 2-12                     [32, 256]                 --
│    └─Linear: 2-13                      [32, 1]                   257
│    └─Sigmoid: 2-14                     [32, 1]                   --
==========================================================================================´´´
Total params: 7,573,505
Trainable params: 7,573,505
Non-trainable params: 0
Total mult-adds (M): 242.35
==========================================================================================
Input size (MB): 0.30
Forward/backward pass size (MB): 0.98
Params size (MB): 30.29
Estimated Total Size (MB): 31.58
==========================================================================================

For the generator I used the following to create a summary and unfortunately I wasnt able to includ ethe column for output shape as well as everything under and including the input row (as above):
model = Generator()
batch_size = 32
summary(model, output_size=(batch_size, 3, 28, 28))

and received the following shorter summary:
=================================================================
Layer (type:depth-idx)                   Param #
=================================================================
Generator                                --
├─Sequential: 1-1                        --
│    └─Linear: 2-1                       25,856
│    └─ReLU: 2-2                         --
│    └─Linear: 2-3                       131,584
│    └─ReLU: 2-4                         --
│    └─Linear: 2-5                       525,312
│    └─ReLU: 2-6                         --
│    └─Linear: 2-7                       2,099,200
│    └─ReLU: 2-8                         --
│    └─Linear: 2-9                       4,819,248
│    └─Tanh: 2-10                        --
=================================================================
Total params: 7,601,200
Trainable params: 7,601,200
Non-trainable params: 0
=================================================================

Output:
This package is not recommended for debugging your code, you should therefore always make sure your code runs on random data before testing the summary.
In the second group of commands, you are using output_size instead of input_size (cf. src). Looking at your code for Generator, the input shape should be (batch_size, 100). Additionally your final linear layer should output a total of 3*28*28 values in order for you to reshape to an image of shape (3, 28, 28).
class Generator(nn.Module):
    def __init__(self):
        super().__init__()
        self.model = nn.Sequential(
            nn.Linear(100, 256),
            nn.ReLU(),
            nn.Linear(256, 512),
            nn.ReLU(),
            nn.Linear(512, 1024),
            nn.ReLU(),
            nn.Linear(1024, 2048),
            nn.ReLU(),
            nn.Linear(2048, 28*28*3),
            nn.Tanh(),
        )

    def forward(self, x):
        output = self.model(x)
        output = output.view(x.size(0), 3, 28, 28)
        return output

Which you can summarize with:
>>> summary(model, input_size=(10,100))
========================================================================================
Layer (type:depth-idx)                   Output Shape              Param #
========================================================================================
Generator                                [10, 3, 28, 28]           --
├─Sequential: 1-1                        [10, 2352]                --
│    └─Linear: 2-1                       [10, 256]                 25,856
│    └─ReLU: 2-2                         [10, 256]                 --
│    └─Linear: 2-3                       [10, 512]                 131,584
│    └─ReLU: 2-4                         [10, 512]                 --
│    └─Linear: 2-5                       [10, 1024]                525,312
│    └─ReLU: 2-6                         [10, 1024]                --
│    └─Linear: 2-7                       [10, 2048]                2,099,200
│    └─ReLU: 2-8                         [10, 2048]                --
│    └─Linear: 2-9                       [10, 2352]                4,819,248
│    └─Tanh: 2-10                        [10, 2352]                --
========================================================================================
Total params: 7,601,200
Trainable params: 7,601,200
Non-trainable params: 0
Total mult-adds (M): 76.01
========================================================================================
Input size (MB): 0.00
Forward/backward pass size (MB): 0.50
Params size (MB): 30.40
Estimated Total Size (MB): 30.90
========================================================================================


Instruction: How can I access the next step data using DataLoader in PyTorch?

Input:
I am using a code that trains neural networks. The code uses the DataLoader of PyTorch to load the data for every iteration. The code looks as follows
for step, data in enumerate(dataloader, 0):
      ............................................................
      output = neuralnetwork_model(data)
      .............................................................

Here the step is an integer that gives values 0, 1, 2, 3, ....... and data gives a batch of samples at each step. The code passes corresponding batches to the neural network at each step.
I need to just access the data of step n+1 at step n. I need something like this
for step, data in enumerate(dataloader, 0):
      ............................................................
      output = neuralnetwork_model(data)
      access = data_of_next_step
      .............................................................

How can I achieve this?

Output:
It seems to be handier to perform such manipulation at the iteration level rather than having to change the data loaders implementation. Looking at Iterate over n successive elements with overlap you can achieve this using itertools.tee:
def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return zip(a, b)

Therefore you simply have to iterate over your wrapped data loader with:
>>> for batch1, batch2 pairwise(dataloader)
...     # batch1 is current batch
...     # batch2 is batch of following step


Instruction: PyG: Remove existing edges from prediction matrix

Input:
I'm currently working on a recommender system using PyG.
The edges are defined as follows:
edge_index = tensor([[   0,    0,    0,  ..., 9315, 9317, 9317],
         [ 100,  448,  452,  ...,  452,    1,  307]], device='cuda:0')}

edge_index[0] containing the indexes for a student and edge_index[1] containing the index of connected modules (both have the same length). Therefore edge_index[0][i] is the source node of edge i and edge_index[1][i] is the destination of edge i.
After model training, I'm generating a 2D-tensor recs with the shape of # of Students x # of Modules, with values from 0-1. 0 = not recommended and 1 = recommended.  recs could look like this:
recs = tensor([0.54, 0.23, 0.98, ..., 0.12, 0.43, 0.87],
              ...,
              [0.43, 0.53, 0.12, ..., 0.92, 0.12, 0.53])

Of course, I don't want to recommend a module if the student has already taken it. Is there a way to set all edges from the original graph to zero, by using the edge_index from PyG as coordinates or something?
Basically i want to set specific values in recs to 0 like this:
for i in range(0, len(edge_index[0])):
  recs[edge_index[0][i]][edge_index[1][i]] = 0

Is there a way using the tensor functions to achieve this?

Output:
Since you want to index recs on both axes simultaneously a straight implementation is to to vectorize your for loop as:
>>> recs[edge_index[0], edge_index[1]] = 0

Which you can improve by splitting edge_index with tuple:
>>> recs[tuple(edge_index)] = 0


Instruction: Are those Keras and PyTorch snippets equivalent?

Input:
I am wondering if I succeeded in translating the following definition in PyTorch to Keras?
In PyTorch, the following multi-layer perceptron was defined:
from torch import nn
hidden = 128
def mlp(size_in, size_out, act=nn.ReLU):
    return nn.Sequential(
        nn.Linear(size_in, hidden),
        act(),
        nn.Linear(hidden, hidden),
        act(),
        nn.Linear(hidden, hidden),
        act(),
        nn.Linear(hidden, size_out),
    )

My translation is
from tensorflow import keras

from keras import layers

hidden = 128

def mlp(size_in, size_out, act=keras.layers.ReLU):
    return keras.Sequential(
        [
            layers.Dense(hidden, activation=None, name="layer1", input_shape=(size_in, 1)),
            act(),
            layers.Dense(hidden, activation=None, name="layer2", input_shape=(hidden, 1)),
            act(),
            layers.Dense(hidden, activation=None, name="layer3", input_shape=(hidden, 1)),
            act(),
            layers.Dense(size_out, activation=None, name="layer4", input_shape=(hidden, 1))
        ])

I am particularly confused about the input/output arguments, because that seems to be where tensorflow and PyTorch differ.
From the documentation:

When a popular kwarg input_shape is passed, then keras will create an
input layer to insert before the current layer. This can be treated
equivalent to explicitly defining an InputLayer.

So, did I get it right?

Output:
In Keras, you can provide an input_shape for the first layer or alternatively use the tf.keras.layers.Input layer. If you do not provide either of these details, the model gets built the first time you call fit, eval, or predict, or the first time you call the model on some input data. So the input shape will actually be inferred if you do not provide it. See the docs for more details. PyTorch generally infers the input shape at runtime.
def keras_mlp(size_in, size_out, act=layers.ReLU):
    return keras.Sequential([layers.Input(shape=(size_in,)),
                             layers.Dense(hidden, name='layer1'),
                             act(),
                             layers.Dense(hidden, name='layer2'),
                             act(),
                             layers.Dense(hidden, name='layer3'),
                             act(),
                             layers.Dense(size_out, name='layer4')])

def pytorch_mlp(size_in, size_out, act=nn.ReLU):
    return nn.Sequential(nn.Linear(size_in, hidden),
                         act(),
                         nn.Linear(hidden, hidden),
                         act(),
                         nn.Linear(hidden, hidden),
                         act(),
                         nn.Linear(hidden, size_out))

You can compare their summary.

For Keras:
>>> keras_mlp(10, 5).summary()
Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #
=================================================================
 layer1 (Dense)              (None, 128)               1408

 re_lu_6 (ReLU)              (None, 128)               0

 layer2 (Dense)              (None, 128)               16512

 re_lu_7 (ReLU)              (None, 128)               0

 layer3 (Dense)              (None, 128)               16512

 re_lu_8 (ReLU)              (None, 128)               0

 layer4 (Dense)              (None, 5)                 645

=================================================================
Total params: 35,077
Trainable params: 35,077
Non-trainable params: 0
_________________________________________________________________

For PyTorch:
>>> summary(pytorch_mlp(10, 5), (1,10))
============================================================================
Layer (type:depth-idx)                   Output Shape              Param #
============================================================================
Sequential                               [1, 5]                    --
├─Linear: 1-1                            [1, 128]                  1,408
├─ReLU: 1-2                              [1, 128]                  --
├─Linear: 1-3                            [1, 128]                  16,512
├─ReLU: 1-4                              [1, 128]                  --
├─Linear: 1-5                            [1, 128]                  16,512
├─ReLU: 1-6                              [1, 128]                  --
├─Linear: 1-7                            [1, 5]                    645
============================================================================
Total params: 35,077
Trainable params: 35,077
Non-trainable params: 0
Total mult-adds (M): 0.04
============================================================================
Input size (MB): 0.00
Forward/backward pass size (MB): 0.00
Params size (MB): 0.14
Estimated Total Size (MB): 0.14
============================================================================


Instruction: I am new to pytorch , why am I getting the attribute error even after adding super(ClassName,self).__init__()

Input:
I am working on a chatbot based out of pytorch
I am unable to figure out the reason behind the attribute error even after adding super to the NueralNet class.
Using jupyter notebook for this project.
HERE IS MY CODE SO FAR , parts from both model.ipynb and train.ipynb

class NeuralNet(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super().__init__()
        self.l1 = nn.Linear(input_size, hidden_size)
        self.l2 = nn.Linear(hidden_size, hidden_size)
        self.l3 = nn.Linear(hidden_size, num_classes)
        self.relu = nn.ReLU()

#Train

import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader

from ipynb.fs.full.model import NeuralNet

class ChatDataset(Dataset):

    def __init__(self):
        self.n_samples = len(x_train)
        self.x_data = x_train
        self.y_data = y_train

#hyperParameters
batch_size = 8
hidden_size = 8
output_size = len(tags)
input_size = len(all_words)
# print(input_size, len(all_words))
# print(output_size, tags)
learning_rate = 0.001
num_epochs = 1000

dataset = ChatDataset()
train_loader = DataLoader(dataset = dataset , batch_size=batch_size, shuffle=True, num_workers=2)
model = NeuralNet(input_size, hidden_size, output_size)```

ERROR:

~\Untitled Folder\model.ipynb in __init__(self, input_size, hidden_size, num_classes)
      9    "source": [
     10     "import torch\n",
---> 11     "import torch.nn as nn"
     12    ]
     13   },

~\anaconda3a\lib\site-packages\torch\nn\modules\module.py in __setattr__(self, name, value)
   1234             if isinstance(value, Module):
   1235                 if modules is None:
-> 1236                     raise AttributeError(
   1237                         "cannot assign module before Module.__init__() call")
   1238                 remove_from(self.__dict__, self._parameters, self._buffers, self._non_persistent_buffers_set)

AttributeError: cannot assign module before Module.__init__() call

Output:
This is most probably because you haven't called super().__init__ in your __init__ function of NeuralNet before registering sub-modules to it. See here for additional details.
The only missing component is a function __len__ on ChatDataset. Other than that the provided code runs fine.


Instruction: Do Layer Normalization in Pytorch without learnable parameters?

Input:
We can add layer normalization in Pytorch by doing: torch.nn.LayerNorm(shape). However, this is layer normalization with learnable parameters. I.e, it's the following equation:

Does Pytorch have builtin layer normalization without learnable parameters?

Output: You can use nn.LayerNorm, setting the elementwise flag to False. This way the layer won't have learnt parameters. See source code for additional reference.


Instruction: Pytorch identifying batch size as number of channels in Conv2d layer

Input:
I am a total newbie to neural networks using Pytorch to create a VAE model. I've used a bit of tensorflow before, but I have no idea what "in_channels" and "out_channels" are, as arguments to nn.Conv2d/nn.Conv1d.
Disclaimers aside, currently, my model takes in a dataloader with batch size 128 and where each input is a 248 by 46 tensor (so, a 128 x 248 x 46 tensor).
My encoder looks like this right now -- I chopped it down so I could focus on where the error was coming from.
class Encoder(nn.Module):
    def __init__(self, latent_dim):
        super(Encoder, self).__init__()
        self.latent_dim = latent_dim
        self.conv1 = nn.Conv2d(in_channels=248, out_channels=46, kernel_size=(9, 9), stride=(5, 1), padding=(5, 4))

    def forward(self, x):
        print(x.size())
        x = F.relu(self.conv1(x))
        return x

The Conv2d layer was meant to reduce the 248 by 46 input into a 50 by 46 tensor. However, I get this error:
RuntimeError: Given groups=1, weight of size [46, 248, 9, 9], expected input[1, 128, 248, 46] to have 248 channels, but got 128 channels instead

...even though I print x.size() and it displays as [torch.Size([128, 248, 46]).
I am unsure a) why the error shows that the layer is adding on an extra dimension to x, and b) whether I am even understanding channels correctly. Should 46 be the real number of channels? Why doesn't Pytorch simply request my input size as a tuple or something, like in=(248, 46)?
Or c) if this is an issue with the way I loaded in my data to the model. I have a numpy array data of shape (-1, 248, 46) and then started training my model as follows.
tensor_data = torch.from_numpy(data)
dataset = TensorDataset(tensor_data, tensor_data)
train_dl = DataLoader(dataset, batch_size=128, shuffle=True)
...
for epoch in range(20):
     for x_train, y_train in train_loader:
          x_train = x_train.to(device).float()
          optimizer.zero_grad()
          x_pred, mu, log_var = vae(x_train)
          bce_loss = train.BCE(y_train, x_pred)
          kl_loss = train.KL(mu, log_var)
          loss = bce_loss + kl_loss
          loss.backward()
          optimizer.step()

Any thoughts appreciated!

Output:
In pytorch, nn.Conv2d assumes the input (mostly image data) is shaped like: [B, C_in, H, W], where B is the batch size, C_in is the number of channels, H and W are the height and width of the image. The output has a similar shape [B, C_out, H_out, W_out]. Here, C_in and C_out are in_channels and out_channels, respectively. (H_out, W_out) is the output image size, which may or may not equal (H, W), depending on the kernel size, the stride and the padding.
However, it is confusing to apply conv2d to reduce [128, 248, 46] inputs to [128, 50, 46]. Are they image data with height 248 and width 46? If so you can reshape the inputs to [128, 1, 248, 46] and use in_channels = 1 and out_channels = 1 in conv2d.


Instruction: Overwriting vs mutating pytorch weights

Input:
I'm trying to understand why I cannot directly overwrite the weights of a torch layer.
Consider the following example:
import torch
from torch import nn

net = nn.Linear(3, 1)
weights = torch.zeros(1,3)

# Overwriting does not work
net.state_dict()["weight"] = weights  # nothing happens
print(f"{net.state_dict()['weight']=}")

# But mutating does work
net.state_dict()["weight"][0] = weights  # indexing works
print(f"{net.state_dict()['weight']=}")

#########
# output
: net.state_dict()['weight']=tensor([[ 0.5464, -0.4110, -0.1063]])
: net.state_dict()['weight']=tensor([[0., 0., 0.]])

I'm confused since state_dict()["weight"] is just a torch tensor, so I feel I'm missing something really obvious here.

Output:
This is because net.state_dict() first creates a collections.OrderedDict object, then stores the weight tensor(s) of this module to it, and returns the dict:
state_dict = net.state_dict()
print(type(state_dict))    # <class 'collections.OrderedDict'>

When you "overwrite" (it's in fact not an overwrite; it's assignment in python) this ordered dict, you reassign an int 0 to the key 'weights' of this ordered dict. The data in that tensor is not modified, it's just not referred to by the ordered dict.
When you check whether the tensor is modified by:
print(f"{net.state_dict()['weight']}")

a new ordered dict different from the one you have modified is created, so you see the unchanged tensor.
However, when you use indexing like this:
net.state_dict()["weight"][0] = weights  # indexing works

then it's not assignment to the ordered dict anymore. Instead, the __setitem__ method of the tensor is called, which allows you to access and modify the underlying memory inplace. Other tensor APIs such as copy_ can also achieve desired results.
A clear explanation on the difference of a = b and a[:] = b when a is a tensor/array can be found here: https://stackoverflow.com/a/68978622/11790637


Instruction: PyTorch Dataset / Dataloader from random source

Input:
I have a source of random (non-deterministic, non-repeatable) data, that I'd like to wrap in Dataset and Dataloader for PyTorch training.  How can I do this?
__len__ is not defined, as the source is infinite (with possible repition).
__getitem__ is not defined, as the source is non-deterministic.

Output:
When defining a custom dataset class, you'd ordinarily subclass torch.utils.data.Dataset and define __len__() and __getitem__().
However, for cases where you want sequential but not random access, you can use an iterable-style dataset. To do this, you instead subclass torch.utils.data.IterableDataset and define __iter__(). Whatever is returned by __iter__() should be a proper iterator; it should maintain state (if necessary) and define __next__() to obtain the next item in the sequence. __next__() should raise StopIteration when there's nothing left to read. In your case with an infinite dataset, it never needs to do this.
Here's an example:
import torch

class MyInfiniteIterator:
    def __next__(self):
        return torch.randn(10)

class MyInfiniteDataset(torch.utils.data.IterableDataset):
    def __iter__(self):
        return MyInfiniteIterator()

dataset = MyInfiniteDataset()
dataloader = torch.utils.data.DataLoader(dataset, batch_size = 32)

for batch in dataloader:
    # ... Do some stuff here ...
    # ...

    # if some_condition:
    #     break


Instruction: convert from 3 channels to 1 channel pytorch tensor from list of tensors

Input:
Say I have a list of tensors, volumes, which I can iterate over:
    for volume in range(len(volumes)):
        print (volume.shape)

torch.Size([3, 512, 512, 222])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 185])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 271])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 261])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 215])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 284])
<class 'torch.Tensor'>
torch.Size([3, 512, 512, 191])
<class 'torch.Tensor'>

How can I change the channel from 3 to 1, for all volumes?
Thanks

Output:
If you like just to keep the first channel for each volume, you can create a new list like that:
new_volumes = [volume[0,...] for volume in volumes]


Instruction: Pytorch/Numpy: Subtract each of N elements from a single matrix, resulting in N matrices?

Input:
Question in the title. Is there an operation or way to broadcast to do this without looping? Here's a simple example with list comprehension:
image = torch.tensor([[6, 9], [8.7, 5.5]])
c = torch.tensor([5.7675, 8.8325])

# with list comprehension
desired_result = torch.stack([image - c_i for c_i in c])

# output:
tensor([[[ 0.2325,  3.2325],
         [ 2.9325, -0.2675]],

        [[-2.8325,  0.1675],
         [-0.1325, -3.3325]]])

I've tried reshaping the "scalar array" every which way to get the desired results with no luck.

Output:
Not sure if torch has outer:
- np.subtract.outer(c.numpy(), image.numpy() )

Output:
array([[[ 0.23250008,  3.2325    ],
        [ 2.9325    , -0.26749992]],

       [[-2.8325005 ,  0.16749954],
        [-0.13250065, -3.3325005 ]]], dtype=float32)

In torch, you can flatten the two tensors and reshape:
-(c[:,None] - image.ravel()).reshape(*c.shape, *image.shape)

Output:
tensor([[[ 0.2325,  3.2325],
         [ 2.9325, -0.2675]],

        [[-2.8325,  0.1675],
         [-0.1325, -3.3325]]])


Instruction: Detectron2 code not showing anything in object detection

Input:
I am using the code described in this article for running inference (object detection) on an image using a trained model.
# import some common detectron2 utilities
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog
import cv2

from detectron2.data.detection_utils import read_image

WINDOW_NAME = "COCO detections"
#im = cv2.imread("sample.jpg")  # this also shows the same result
im = read_image("sample.jpg", format="BGR")

# Create config
cfg = get_cfg()
cfg.merge_from_file("C:/Users/preet/detectron_repo/configs/COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x.yaml")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
cfg.MODEL.WEIGHTS = "detectron2://COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x/139173657/model_final_68b088.pkl"

# Create predictor
predictor = DefaultPredictor(cfg)

# Make prediction
outputs = predictor(im)

v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)
v = v.draw_instance_predictions(outputs["instances"].to("cpu"))
cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
cv2.imshow(WINDOW_NAME, v.get_image()[:, :, ::-1])

But the window that pops out shows "not responding" and I don't see anything in that window. Moreover, I get the following warning:
UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at  C:\cb\pytorch_1000000000000\work\aten\src\ATen\native\TensorShape.cpp:2228.)
  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]

How to resolve this?

Output:
After digging out, I found out that I missed this line at the end:
cv2.waitKey(0)

That's why everything was working perfectly but the window wasn't responding.


Instruction: PyTorch index in a batch

Input:
Given tensor IN of shape (A, B, C, D) and index tensor IDX of shape [A, B, C] with torch.long values in [0, C), how can I get a tensor OUT of shape (A, B, C, D) such that:
OUT[a, b, c, :] == IN[a, b, IDX[a, b, c], :]

This is trivial without dimensions A and B:
# C = 2, D = 3
IN = torch.arange(6).view(2, 3)
IDX = torch.tensor([0,0])

print(IN[IDX])
# tensor([[0, 1, 2],
#         [0, 1, 2]])

Obviously, I can write a nested for loop over A and B. But surely there must be a vectorized way to do it?

Output:
This is the perfect use case for torch.gather. Given two 4d tensors, input the input tensor and index the tensor containing the indices for input, calling torch.gather on dim=2 will return a tensor out shaped like input such that:
out[i][j][k][l] = input[i][j][index[i][j][k][l]][l]

In other words, index indexes dimension n°3 of input.
Before applying such function though, notice all tensors must have the same number of dimensions. Since index is only 3d, we need to insert and expand an additional 4th dimension on it. We can do so with the following lines:
>>> idx_ = idx[...,None].expand_as(x)

Then call the torch.gather function
>>> x.gather(dim=2, index=idx_)

You can try out the solution with this code:
>>> A = 1; B = 2; C=3; D=2

>>> x = torch.rand(A,B,C,D)
tensor([[[[0.6490, 0.7670],
          [0.7847, 0.9058],
          [0.3606, 0.7843]],

         [[0.0666, 0.7306],
          [0.1923, 0.3513],
          [0.5287, 0.3680]]]])

>>> idx = torch.randint(0, C, (A,B,C))
tensor([[[1, 2, 2],
         [0, 0, 1]]])

>>> x.gather(dim=2, index=idx[...,None].expand_as(x))
tensor([[[[0.7847, 0.9058],
          [0.3606, 0.7843],
          [0.3606, 0.7843]],

         [[0.0666, 0.7306],
          [0.0666, 0.7306],
          [0.1923, 0.3513]]]])


Instruction: how to convert .pth.tar file to a .pt?

Input:
I have a .pth.tar file how can I read in that file and then save it to the same directory in a .pt file?
I am using the modelzoo mlnf file which I cannot find its .pt equivalent.
https://kaiyangzhou.github.io/deep-person-reid/MODEL_ZOO.html

Output:
It turns out you can load your .pth.tar file directly with torch.load:
state_dict = torch.load(model_name)

See this thread for reference.


Instruction: Weird Python Lambda() syntax

Input:
As below, I understand lambda y:... .
But the first Lambda(...) is a function?.
ds = datasets.FashionMNIST(
...
    target_transform=Lambda(lambda y: torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))
)

Output: It's just a function in torchvision for wrapping an arbitrary function as a transform.  It's nothing to do with Python syntax, and is spelled Lambda with a capital L instead of lambda to not conflict with the Python keyword.


Instruction: How does dim argument of "Tensor.scatter_" method in PyTorch work?

Input:
Could anyone teach me why the below code uses dim=1 in the scatter_ method? The meaning of the attached codes is for one-hot encoding. I tried to read the PyTorch document example and thought I should use dim=0 for the desired result. However, the result has shown that dim=1 is correct instead.
>>> target = torch.tensor([3, 5, 0, 2, 7, 5])
>>> target
tensor([3, 5, 0, 2, 7, 5])

>>> onehot = torch.zeros(target.shape[0], 8)
>>> onehot.scatter_(1, target.unsqueeze(1), 1.0)
tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 1.],
        [0., 0., 0., 0., 0., 1., 0., 0.]])

Output:
You are applying scatter on a zero tensor onehot shaped (len(target), 8) on dim=1 using target as input and 1. as value. This will have the following effect on onehot:
onehot[i][target[i][j]] = 1.

This means for every row in target it will look at the unique value since j is always equal to 1 and use it to index the 2nd axis of onehot. In other words, for every row, it takes the value from target to position the 1. among the columns of onehot.
Step by step illustration would be:
>>> for i in range(len(target)):
...     k = target[i] # k, depends on values of target i.e. dim=1
...     onehot[i, k] = 1
...     print(onehot)

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],  # i=0; k=3
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.]])

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],  # i=1; k=5
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.]])

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0.],  # i=2; k=0
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.]])

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 0., 0., 0.],  # i=3; k=2
        [0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0.]])

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 1.],  # i=4; k=7
        [0., 0., 0., 0., 0., 0., 0., 0.]])

tensor([[0., 0., 0., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 1., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 1.],
        [0., 0., 0., 0., 0., 1., 0., 0.]]) # i=5; k=5

Notice that onehot.scatter_(0, target.unsqueeze(1), 1.0) would have produced:
onehot[target[i][j]][j] = 1.

Which is a valid operation only if you initialize onehot the other way around:
>>> onehot = torch.zeros(8, len(target))
>>> onehot.scatter_(0, target.unsqueeze(1), 1.)
tensor([[1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0.]])

And you get the transpose of the other matrix.


Instruction: Why is my generator and discriminator loss converging at higher values in WGAN-GP?

Input:
This is the loss plot of WGAN-GP after training for 14000 iterations. My image size is 128 by 128. Though the loss plot seems to be converging, the generator loss at iteration 14000 is -26646 and critic loss is -249909.
Loss plot

Output:
Batch Normalization in the discriminator breaks Wasserstein GANs with gradient penalty. The authors themselves advocate the usage of layer normalization instead, but this is clearly written in bold in their paper (https://papers.nips.cc/paper/7159-improved-training-of-wasserstein-gans.pdf). It is hard to say if there are other bugs in your code, but I urge you to thoroughly read the DCGAN and the Wasserstein GAN paper and really take notes on the hyperparameters. Getting them wrong really destroys the performance of the GAN and doing a hyperparameter search gets expensive quite quickly.
By the way transposed convolutions produce stairway artifacts in your output images. Use image resizing instead. For an indepth explanation of that phenomenon I can recommend the following resource (https://distill.pub/2016/deconv-checkerboard/).
This is an interesting find as well, which may help you:  Accelerated WGAN update strategy with loss change rate balancing.


Instruction: pytorch conv2d vs numpy results are different

Input:
I'm working on implementing pytorch conv2d with numpy. But pytorch conv2d vs numpy results are different for the same input and conv weight. How to fix it? Thanks for any help.
Code sample below:
Note:
The code contains 4 parts:

Fixed random seed to generate fixed input and conv weight.
Implement conv2d with pytorch.
Implement conv2d with numpy.
Run conv2d and verify the results are different.

import random
import numpy as np
import torch
import torch.nn as nn

fixed_seed = 5179
np.random.seed(fixed_seed)
random.seed(fixed_seed)
torch.manual_seed(fixed_seed)

np.set_printoptions(precision=8, floatmode="fixed")
torch.set_printoptions(precision=8)

def conv_forward_torch(input_image_tensor, weight, stride, pad):
    # input_image_tensor B  Hi Wi Ci
    # weight             Hk Wk Ci Co
    input_image_tensor = input_image_tensor.permute(0, 3, 1, 2) # B Ci Hi Wi
    weight = weight.permute(3, 2, 0, 1) # Co Ci Hk Wk
    output = torch.nn.functional.conv2d(input_image_tensor, weight, stride=stride, padding=pad) # B Co Ho Wo
    output = output.permute(0, 2, 3, 1).cpu().detach().numpy()  # B Ho Wo Co
    return output

def conv_forward_naive(x, w, stride, pad, bias = None):
    # x B  Hi Wi Ci
    # w Hk Wk Ci Co
    x = np.transpose(x, [0, 3, 1, 2]) # B  Ci Hi Wi
    w = np.transpose(w, [3, 2, 0, 1]) # Co Ci Kh Kw

    if pad != 0:
        x = np.pad(x, ((0, 0), (0 ,0), (pad, pad), (pad, pad)),'constant')

    b,  ci, hi, wi = x.shape
    co, ci, hk, wk = w.shape
    ho = np.floor(1 + (hi - hk) / stride).astype(int)
    wo = np.floor(1 + (wi - wk) / stride).astype(int)
    out = np.zeros((b, co, ho, wo), dtype=np.float32) # B Co Ho Wo

    x = np.expand_dims(x, axis=1) # B 1  Ci Hi Wi
    w = np.expand_dims(w, axis=0) # 1 Co Ci Hk Wk

    for i in range(ho):
        for j in range(wo):
            x_windows = x[:, :, :, i * stride:i * stride + hk, j * stride: j * stride + wk] # B 1 Ci Hk Wk
            out[:, :, i, j] = np.sum(x_windows * w, axis=(2, 3, 4)) # B Co

    out = np.transpose(out, [0, 2, 3, 1]) # B Ho Wo Co
    return out

B  = 1 # Batch size
Hi = 2 # Input height
Wi = 2 # Input width
Ci = 1 # Input channel
Co = 1 # Ouput channel

P = 0 # Padding size

Hk = 2 # Kernel height
Wk = 2 # Kernel width

S = 1 # Stride

input_image_tensor = torch.randn(B, Hi, Wi, Ci)
conv_weight_tensor = torch.randn(Hk, Wk, Ci, Co)

input_image = input_image_tensor.detach().numpy()
conv_weight = conv_weight_tensor.detach().numpy()

y_torch = conv_forward_torch(input_image_tensor, conv_weight_tensor, S, P)
y_np = conv_forward_naive(input_image, conv_weight, S, P)

is_same = y_torch == y_np

print(is_same, y_torch, y_np)

Expected output:
is_same should be True.
Actual output for y_torch, y_np:
[[[[False]]]] [[[[-3.62229419]]]] [[[[-3.62229395]]]]
torch.version = '1.11.0'
np.version = '1.20.1'

Output:
Your code works as expected. The result you observed is because of a difference in floating-point precisions between NumPy and PyTorch. To compare floating points you should not use a direct equal check, but instead something like np.allclose.
In this case your snippet indeed returns True:
>>> np.allclose(y_torch, y_np)
True


Instruction: Create 3D kernels with Pytorch: Is there a simple way for large kernels?

Input:
I would like to test multiple kernels for convolution in 3D space. Here k is a 7x7x7 cross kernel, where the voxel in the middle of the kernel equals 1 and all other voxels in the wings equal -1. This way of assigning k is messy and time-consuming. Any ideas to make it more clear and simple?
tensor([[[[[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0., -1.,  0.,  0.,  0.],
           [ 0.,  0.,  0., -1.,  0.,  0.,  0.],
           [ 0.,  0.,  0., -1.,  0.,  0.,  0.],
           [-1., -1., -1.,  1., -1., -1., -1.],
           [ 0.,  0.,  0., -1.,  0.,  0.,  0.],
           [ 0.,  0.,  0., -1.,  0.,  0.,  0.],
           [ 0.,  0.,  0., -1.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]],

          [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  1.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.,  0.,  0.]]]]])

Output:
You can do so with column and row assignments, given kernel_size, in your case 7. Keep in mind that the shape of a nn.Conv3d layer weight is (out_channels=n_filters, in_channels, kernel_size_depth, kernel_size_height, kernel_size_width).
>>> n = kernel_size // 2
>>> k = torch.zeros(1,1,*(kernel_size,)*3) # define zero tensor

>>> k[...,n,n,:] = -1 # middle column at middle depth
>>> k[...,n,:,n] = -1 # middle row at middle depth
>>> k[...,n,n] = 1    # middle point at all depths


Instruction: How do you use Pytorch model's function in Onnx to get output instead of model.forward() function

Input:
TL;DR: How can I use model.whatever_function(input) instead of model.forward(input) for the onnxruntime?
I use CLIP embedding to create embedding for my Image and texts as:
Code is from the official git merge
! pip install ftfy regex tqdm
! pip install git+https://github.com/openai/CLIP.git

import clip
import torch

device = 'cuda' if torch.cuda.is_available() else 'cpu'

model, preprocess = clip.load("RN50", device=device) # Load any model
model = model.eval() # Inference Only

img_size = model.visual.input_resolution
dummy_image = torch.randn(10, 3, img_size, img_size).to(device)
image_embedding = model.encode_image(dummy_image).to(device))

dummy_texts = clip.tokenize(["quick brown fox", "lorem ipsum"]).to(device)
model.encode_text(dummy_texts)

and it works fine giving me [Batch, 1024] tensors for both for the loaded model.
Now I have quantized my model in Onnx as:
model.forward(dummy_image,dummy_texts) # Original CLIP result (1)

torch.onnx.export(model, (dummy_image, dummy_texts), "model.onnx", export_params=True,
  input_names=["IMAGE", "TEXT"],
  output_names=["LOGITS_PER_IMAGE", "LOGITS_PER_TEXT"],
  opset_version=14,
  dynamic_axes={
      "IMAGE": {
          0: "image_batch_size",
      },
      "TEXT": {
          0: "text_batch_size",
      },
      "LOGITS_PER_IMAGE": {
          0: "image_batch_size",
          1: "text_batch_size",
      },
      "LOGITS_PER_TEXT": {
          0: "text_batch_size",
          1: "image_batch_size",
      },
  }
)

and the model is saved.
When I test the model as :
# Now run onnxruntime to verify
import onnxruntime as ort

ort_sess = ort.InferenceSession("model.onnx")
result=ort_sess.run(["LOGITS_PER_IMAGE", "LOGITS_PER_TEXT"],
  {"IMAGE": dummy_image.numpy(), "TEXT": dummy_texts.numpy()})

It gives me a list of length 2, one for each image and text and the result[0] has shape of [Batch,2].

Output:
If your encode_image on your module isn't calling forward then nothing is stopping you from overriding forward before exporting to Onnx:
>>> model.forward = model.encode_image
>>> torch.onnx.export(model, (dummy_image, dummy_texts), "model.onnx", ...))


Instruction: Vectorizing torch tensor instead of using for loop

Input:
I am looking to make this calculation without using any for loops (vectorized) but cant really seem to find a good solution. Maybe someone can help?
    edge_in = torch.ones(len(edge_embeds), len(edge_embeds[0]), len(edge_embeds[0][0]) + 2*len(nodes_a_embeds[0]))

    for i in range(0, len(nodes_a_embeds)): # A
      for u in range(0, len(nodes_b_embeds)): # B
        edge_in[i][u] = torch.cat([nodes_a_embeds[i], nodes_b_embeds[u], edge_embeds[i][u]], dim=0)

    # OUT: edge_in: torch.Tensor with shape (|A|, |B|, 2*node_dim + 2*edge_dim)

    # IN: edge_embeds: torch.Tensor with shape (|A|, |B|, 2 x edge_dim)
    # IN: nodes_a_embeds: torch.Tensor with shape (|A|, node_dim)
    # IN: nodes_b_embeds: torch.Tensor with shape (|B|, node_dim)

Output:
You can expand nodes_a_embed and nodes_b_embeds to the same shape as edge_embeds and concatenate them directly:

nodes_a_embed = nodes_a_embeds[:, None].expand(-1, n_B, -1): [n_A, node_dim] => [n_A, n_B, node_dim]
nodes_b_embed = nodes_b_embeds[None].expand(n_A, -1, -1): [n_B, node_dim] => [n_A, n_B, node_dim]

Verification:
import torch

n_A = 100
n_B = 200
node_dim = 32
edge_dim = 32

edge_in = torch.randn(n_A, n_B, 2*node_dim + 2*edge_dim)

edge_embeds = torch.randn(n_A, n_B, 2*edge_dim)
nodes_a_embeds = torch.randn(n_A, node_dim)
nodes_b_embeds = torch.randn(n_B, node_dim)

edge_in = torch.ones(len(edge_embeds), len(edge_embeds[0]), len(edge_embeds[0][0]) + 2*len(nodes_a_embeds[0]))

for i in range(0, len(nodes_a_embeds)): # A
    for u in range(0, len(nodes_b_embeds)): # B
        edge_in[i][u] = torch.cat([nodes_a_embeds[i], nodes_b_embeds[u], edge_embeds[i][u]], dim=0)

# vectorized version
edge_in_vectorized = torch.cat([
                nodes_a_embeds[:, None].expand(-1, n_B, -1),
                nodes_b_embeds[None].expand(n_A, -1, -1),
                edge_embeds], dim=-1)

print((edge_in_vectorized == edge_in).all())    # True


Instruction: What happens in a convolution when the stride is larger than the kernel?

Input:
I recently was experiment with convolutions and transposed convolutions in Pytorch. I noticed with the nn.ConvTranspose2d API (I haven't tried with the normal convolution API yet), you can specify a stride that is larger than the kernel size and the convolution will still work.
What is happening in this case? I'm confused because if the stride is larger than the kernel, that means some pixels in the input image will not be convolved. So what happens to them?
I have the following snippet where I manually set the weights for a nn.ConvTranspose2d layer:
IN = 1
OUT = 1
KERNEL_SIZE = 2
proof_conv = nn.ConvTranspose2d(IN, OUT, kernel_size=KERNEL_SIZE, stride=4)
assert proof_conv.weight.shape == (IN, OUT, KERNEL_SIZE, KERNEL_SIZE)

FILTER = [
    [1., 2.],
    [0., 1.]
]
weights = [
    [FILTER]
]

weights_as_tensor = torch.from_numpy(np.asarray(weights)).float()
assert weights_as_tensor.shape == proof_conv.weight.shape
proof_conv.weight = nn.Parameter(weights_as_tensor)

img = [[
  [1., 2.],
  [3., 4.]
]]
img_as_tensor = torch.from_numpy(np.asarray(img)).float()
out_img = proof_conv(img_as_tensor)
assert out_img.shape == (OUT, 6, 6)

The stride is larger than the KERNEL_SIZE of 2. Yet, the transposed convolution still occurs and we get an output of 6x6. What is happening underneath the hood?
This post: Understanding the PyTorch implementation of Conv2DTranspose is helpful but does not answer the edge-case of when the stride is greater than the kernel.

Output:
As you already guessed - when the stride is larger than the kernel size, there are input pixels that do not participate in the convolution operation.
It's up to you - the designer of the architecture to decide whether this property is a bug or a feature. In some cases, I took advantage of this property to ignore portions of the inputs.
Update:
I think you are being confused by the bias term in proof_conv. Try to eliminate it:
proof_conv = nn.ConvTranspose2d(IN, OUT, kernel_size=KERNEL_SIZE, stride=4, bias=False)

Now you'll get out_img to be:
[[[[1., 2., 0., 0., 2., 4.],
          [0., 1., 0., 0., 0., 2.],
          [0., 0., 0., 0., 0., 0.],
          [0., 0., 0., 0., 0., 0.],
          [3., 6., 0., 0., 4., 8.],
          [0., 3., 0., 0., 0., 4.]]]]

Which represent 4 copies of the kernel, weighted by the input image, spaced 4 pixels apart according to stride=4.
The rest of the output image is filled with zeros - representing pixels that do not contribute to the transposed convolution.
ConvTranspose follows the same "logic" as the regular conv, only in a "transposed" fashion. If you look at the formula for computing output shape you'll see that the behavior you get is consistent.


Instruction: How to load the whole dataset to GPU

Input: I have dataset of 1550 images 3x112x112. When i am training my model I create dataset wia ImageFolder and then use DataLoader. It takes so much time because of reading from memory every time. I have enough gpu memory to load the whole dataset at once. It will be much faster. What is the best way to do it?

Output:
You can store the images as an attribute of the dataset, put it on the GPU at initialization, and let __getitem__ return images from this directly.
import torch
from torch.utils.data import Dataset, DataLoader

class GPUDataset(Dataset):
    def __init__(self):
        self.len = 1550
        self.data = torch.randn(self.len, 3, 112, 112).cuda()  # ~ 223 MB

    def __getitem__(self, index):
        return self.data[index]

    def __len__(self):
        return self.len

dataset = GPUDataset()
loader = DataLoader(dataset, batch_size=4)

batch = next(iter(loader))
print(batch.device, batch.shape)    # cuda:0 torch.Size([4, 3, 112, 112])

This is meant to be a quick fix for small datasets. For larger datasets, consider more advanced solutions such as lmdb.


Instruction: `torch.gather` without unbroadcasting

Input:
I have some batched input x of shape [batch, time, feature], and some batched indices i of shape [batch, new_time] which I want to gather into the time dim of x. As output of this operation I want a tensor y of shape [batch, new_time, feature] with values like this:
y[b, t', f] = x[b, i[b, t'], f]

In Tensorflow, I can accomplish this by using the batch_dims: int argument of tf.gather: y = tf.gather(x, i, axis=1, batch_dims=1).
In PyTorch, I can think of some functions which do similar things:

torch.gather of course, but this does not have an argument similar to Tensorflow's batch_dims. The output of torch.gather will always have the same shape as the indices. So I would need to unbroadcast the feature dim into i before passing it to torch.gather.

torch.index_select, but here, the indices must be one-dimensional. So to make it work I would need to unbroadcast x to add a "batch * new_time" dim, and then after torch.index_select reshape the output.

torch.nn.functional.embedding. Here, the embedding matrices would correspond to x. But this embedding function does not support the weights to be batched, so I run into the same issue as for torch.index_select (looking at the code, tf.embedding uses torch.index_select under the hood).

Is it possible to accomplish such gather operation without relying on unbroadcasting which is inefficient for large dims?

Output:
This is actually the most frequent case: when input and index tensors don't perfectly match the number of dimensions. You can still utilize torch.gather though since you can rewrite your expression:
y[b, t, f] = x[b, i[b, t], f]

as:
y[b, t, f] = x[b, i[b, t, f], f]

which ensures all three tensors have an equal number of dimensions. This reveals a third dimension on i, which we can easily create for free by unsqueezing a dimension and expanding it to the shape of x. You can do so with i[:,None].expand_as(x).
Here is a minimal example:
>>> b = 2; t = 3; f = 1
>>> x = torch.rand(b, t, f)
>>> i = torch.randint(0, t, (b, f))

>>> x.gather(1, i[:,None].expand_as(x))


Instruction: bert-base-uncased: TypeError: tuple indices must be integers or slices, not tuple

Input:
I want to see embeddings for the input text I give to the model, and then feed it to the rest of the BERT. To do so, I partitioned the model into two sequential models, but I must have done it wrong because rest_of_bert model raises TypeError. Original model does not raise any error with the input_ids as input processed with text_to_input function.
Input[0]:
import torch
from transformers import BertTokenizer, BertModel

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
cls_token_id = tokenizer.cls_token_id
sep_token_id = tokenizer.sep_token_id
pad_token_id = tokenizer.pad_token_id

model = BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True)
model.eval()

Output[0]:
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
BertModel(
  (embeddings): BertEmbeddings(
    (word_embeddings): Embedding(30522, 768, padding_idx=0)
    (position_embeddings): Embedding(512, 768)
    (token_type_embeddings): Embedding(2, 768)
    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
    (dropout): Dropout(p=0.1, inplace=False)
  )
  (encoder): BertEncoder(
    (layer): ModuleList(
      (0): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (1): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (2): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (3): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (4): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (5): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (6): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (7): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (8): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (9): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (10): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
      (11): BertLayer(
        (attention): BertAttention(
          (self): BertSelfAttention(
            (query): Linear(in_features=768, out_features=768, bias=True)
            (key): Linear(in_features=768, out_features=768, bias=True)
            (value): Linear(in_features=768, out_features=768, bias=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
          (output): BertSelfOutput(
            (dense): Linear(in_features=768, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
        (intermediate): BertIntermediate(
          (dense): Linear(in_features=768, out_features=3072, bias=True)
          (intermediate_act_fn): GELUActivation()
        )
        (output): BertOutput(
          (dense): Linear(in_features=3072, out_features=768, bias=True)
          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
    )
  )
  (pooler): BertPooler(
    (dense): Linear(in_features=768, out_features=768, bias=True)
    (activation): Tanh()
  )
)

Input[1]:
def text_to_input(text):
  x = tokenizer.encode(text, add_special_tokens=False) # returns python list
  x = [cls_token_id] + x + [sep_token_id]
  token_count = len(x)
  pad_count = 512 - token_count
  x = x + [pad_token_id for i in range(pad_count)]
  return torch.tensor([x])

extract_embeddings = torch.nn.Sequential(list(model.children())[0])
rest_of_bert = torch.nn.Sequential(*list(model.children())[1:])

input_ids = text_to_input('A sentence.')
x_embedding = extract_embeddings(input_ids)
output = rest_of_bert(x_embedding)

Output[1]:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-5-d371d8a2fb3c> in <module>()
     12 input_ids = text_to_input('A sentence.')
     13 x_embedding = extract_embeddings(input_ids)
---> 14 output = rest_of_bert(x_embedding)

4 frames
/usr/local/lib/python3.7/dist-packages/transformers/utils/generic.py in __getitem__(self, k)
    220             return inner_dict[k]
    221         else:
--> 222             return self.to_tuple()[k]
    223
    224     def __setattr__(self, name, value):

TypeError: tuple indices must be integers or slices, not tuple

Output:
Each BertLayer returns a tuple that contains at least one tensor (depending on what output you requested). The first element of the tuple is the tensor you want to feed to the next BertLayer.
A more huggingface-like approach would be calling the model with output_hidden_states:
o = model(input_ids, output_hidden_states=True)
print(len(o.hidden_states))

Output:
13

The first tensor of the hidden_states tuple is the output of your extract_embeddings object (token embeddings). The other 12 tensors are the contextualized embeddings that are the output of each BertLayer.
You should, by the way, provide an attention mask, because otherwise, your padding tokens will affect your output. The tokenizer is able to do that for you and you can replace your whole text_to_input method with:
tokenizer('A sentence.', return_tensors='pt', padding='max_length', max_length=512)


Instruction: PyTorch: running Neural Network in identical venv-configuration - one working fine, one keeps throwing 'list index out of range' error

Input:
I set up an virtual environment via 'venv' to build and train a neural network using PyTorch and JupyterLab. However, when working on my computer 'PC 1' everything works fine, but running the same code with identical Python settings on my second computer ('PC 2') keeps constantly an error ('list index out of range') thrown while training.
Any suggestions what might cause this behavior? I'm running out of ideas ...
To be more specific:

On both computers Python 3.7.9 is installed via the Microsoft Store.
To be clear: both computers access the same *.ipynb and the same data / datasets, which are synced via a cloud service.

I tried:

I synced the created venv (via a cloud), activated venv and runned the *.ipynb via jupyter-lab (on 'PC 2') -> error
I got my venv-configuration from the working 'PC 1' via  pip freeze > requirements.txt and set up a fresh venv on 'PC 2' using the requirements.txt-> error

Nevertheless I try, the thrown error is always the same.
This is the error thrown:
('julab' is my venv)
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_9608\2329219183.py in <module>
     10     val_dl = test_loader,
     11     epochs=num_epochs,
---> 12     device='cpu')

~\AppData\Local\Temp\ipykernel_9608\132402798.py in train(model, optimizer, loss_fn, train_dl, val_dl, epochs, device)
     27         num_train_examples = 0
     28
---> 29         for batch in train_dl:
     30
     31             optimizer.zero_grad()

d:\<CLOUD>\<SUBFOLDER>\julab\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    650                 # TODO(https://github.com/pytorch/pytorch/issues/76750)
    651                 self._reset()  # type: ignore[call-arg]
--> 652             data = self._next_data()
    653             self._num_yielded += 1
    654             if self._dataset_kind == _DatasetKind.Iterable and \

d:\<CLOUD>\<SUBFOLDER>\julab\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
    690     def _next_data(self):
    691         index = self._next_index()  # may raise StopIteration
--> 692         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    693         if self._pin_memory:
    694             data = _utils.pin_memory.pin_memory(data, self._pin_memory_device)

d:\<CLOUD>\<SUBFOLDER>\julab\lib\site-packages\torch\utils\data\_utils\fetch.py in fetch(self, possibly_batched_index)
     47     def fetch(self, possibly_batched_index):
     48         if self.auto_collation:
---> 49             data = [self.dataset[idx] for idx in possibly_batched_index]
     50         else:
     51             data = self.dataset[possibly_batched_index]

d:\<CLOUD>\<SUBFOLDER>\julab\lib\site-packages\torch\utils\data\_utils\fetch.py in <listcomp>(.0)
     47     def fetch(self, possibly_batched_index):
     48         if self.auto_collation:
---> 49             data = [self.dataset[idx] for idx in possibly_batched_index]
     50         else:
     51             data = self.dataset[possibly_batched_index]

d:\<CLOUD>\<SUBFOLDER>\julab\lib\site-packages\torch\utils\data\dataset.py in __getitem__(self, idx)
    288         if isinstance(idx, list):
    289             return self.dataset[[self.indices[i] for i in idx]]
--> 290         return self.dataset[self.indices[idx]]
    291
    292     def __len__(self):

~\AppData\Local\Temp\ipykernel_9608\2122586536.py in __getitem__(self, index)
     32
     33     def __getitem__(self, index):
---> 34         image_name = os.path.join(self.image_dir, self.image_files[index])
     35         image = PIL.Image.open(image_name)
     36         label = self.data[index]

IndexError: list index out of range

And as this might help, this is my requirements.txt:
anyio==3.6.1
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
attrs==21.4.0
Babel==2.10.3
backcall==0.2.0
beautifulsoup4==4.11.1
bleach==5.0.1
certifi==2022.6.15
cffi==1.15.1
charset-normalizer==2.1.0
colorama==0.4.5
cycler==0.11.0
debugpy==1.6.0
decorator==5.1.1
defusedxml==0.7.1
dill==0.3.5.1
entrypoints==0.4
fastjsonschema==2.15.3
fonttools==4.33.3
idna==3.3
importlib-metadata==4.12.0
importlib-resources==5.8.0
ipykernel==6.15.0
ipython==7.34.0
ipython-genutils==0.2.0
jedi==0.18.1
Jinja2==3.1.2
joblib==1.1.0
json5==0.9.8
jsonschema==4.6.1
jupyter-client==7.3.4
jupyter-core==4.10.0
jupyter-server==1.18.0
jupyterlab==3.4.3
jupyterlab-pygments==0.2.2
jupyterlab-server==2.14.0
kiwisolver==1.4.3
MarkupSafe==2.1.1
matplotlib==3.5.2
matplotlib-inline==0.1.3
mistune==0.8.4
nbclassic==0.4.0
nbclient==0.6.6
nbconvert==6.5.0
nbformat==5.4.0
nest-asyncio==1.5.5
notebook-shim==0.1.0
numpy==1.21.6
packaging==21.3
pandas==1.3.5
pandocfilters==1.5.0
parso==0.8.3
pickleshare==0.7.5
Pillow==9.2.0
prometheus-client==0.14.1
prompt-toolkit==3.0.30
psutil==5.9.1
pycparser==2.21
Pygments==2.12.0
pyparsing==3.0.9
pyrsistent==0.18.1
python-dateutil==2.8.2
pytz==2022.1
pywin32==304
pywinpty==2.0.5
pyzmq==23.2.0
requests==2.28.1
scikit-learn==1.0.2
scipy==1.7.3
Send2Trash==1.8.0
six==1.16.0
sklearn==0.0
sniffio==1.2.0
soupsieve==2.3.2.post1
terminado==0.15.0
threadpoolctl==3.1.0
tinycss2==1.1.1
torch==1.12.0
torchsummary==1.5.1
torchvision==0.13.0
tornado==6.1
traitlets==5.3.0
typing_extensions==4.3.0
urllib3==1.26.9
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==1.3.3
zipp==3.8.0

Has anyone an idea what's the problem and how to solve it? Many thanks in advance!

Output: [solved] see comments above...


Instruction: Merge images by mask

Input:
I'm trying to merge two images according to the values in a mask where at all points where the mask is 1, the resulting image has the values of the first image, and otherwise it has values of the second image. Does anyone know how it can be achieved in pytorch? Using numpy, it can be achieved using
>>> import numpy as np
>>> img1 = np.random.rand(100,100,3)
>>> img2 = np.random.rand(100,100,3)
>>> mask = np.random.rand(100,100)>.5
>>> res = img2.copy()
>>> res[mask] = img1[mask]

Output:
You are looking for np.where:
res = no.where(mask, img1, img2)


Instruction: Are PyTorch activation functions best stored as fields?

Input:
An example of a simple neural network in PyTorch can be found at https://visualstudiomagazine.com/articles/2020/10/14/pytorch-define-network.aspx
class Net(T.nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.hid1 = T.nn.Linear(4, 8)  # 4-(8-8)-1
    self.hid2 = T.nn.Linear(8, 8)
    self.oupt = T.nn.Linear(8, 1)

    T.nn.init.xavier_uniform_(self.hid1.weight)
    T.nn.init.zeros_(self.hid1.bias)
    T.nn.init.xavier_uniform_(self.hid2.weight)
    T.nn.init.zeros_(self.hid2.bias)
    T.nn.init.xavier_uniform_(self.oupt.weight)
    T.nn.init.zeros_(self.oupt.bias)

  def forward(self, x):
    z = T.tanh(self.hid1(x))
    z = T.tanh(self.hid2(z))
    z = T.sigmoid(self.oupt(z))
    return z

A distinctive feature of the above is that the layers are stored as fields within the Net object (as they need to be, in the sense that they contain the weights, which need to be remembered across training epochs), but the activation functors such as tanh are re-created on every call to forward. The author says:

The most common structure for a binary classification network is to define the network layers and their associated weights and biases in the __init__() method, and the input-output computations in the forward() method.

Fair enough. On the other hand, perhaps it would be marginally faster to store the functors rather than re-create them on every call to forward. On the third hand, it's unlikely to make any measurable difference, which means it might end up being a matter of code style.
Is the above, indeed the most common way to do it? Does either way have any technical advantage, or is it just a matter of style?

Output:
On "storing" functors
The snippet is not "re-creating" anything -- calling torch.tanh(x) is literally just calling the function tanh exported by the torch package with arguments x.

Other ways of doing it
I think the snippet is a fair example for small neural blocks that are use-and-forget or are just not meant to be parameterizable.
Depending on your intentions, there are of course alternatives, but you'd have to weigh yourself whether the added complexity offers any value.

activation functions as strings

allow a selection of an activation function from a fixed set
class Model(torch.nn.Module):
   def __init__(..., activation_function: Literal['tanh'] | Literal['relu']):
       ...
       if activation_function == 'tanh':
           self.activation_function = torch.tanh
       elif activation_function == 'relu':
           self.activation_function = torch.relu
       else:
           raise ValueError(f'activation function {activation_function} not allowed, use tanh or relu.'}

    def forward(...) -> Tensor:
        output = ...
        return self.activation_function(output)

activation functions as callables

use arbitrary modules or functions as activations
class Model(torch.nn.Module):
   def __init__(..., activation_function: torch.nn.Module | Callable[[Tensor], Tensor]):
      self.activation_function = activation_function

   def forward(...) -> Tensor:
       output = ...
       return self.activation_function(output)

which would for instance work like
def cube(x: Tensor) -> Tensor: return x**3

cubic_model = Model(..., activation_function=cube)

The key difference between the above examples and your snippet is the fact that the latter are transparent and adjustable wrt. to the activation used; you can inspect the activation function (i.e. model.activation_function), and change it (before or after initialization), whereas in the case of the original snippet it is invisible and baked into the model's functionality (to replicate the model with a different function, you'd need to define it from scratch).
Overall, I think the best way to go is to create small, locally tunable blocks that are as parametric as you need them to be, and wrap them into bigger blocks that make generalizations over the contained parameters. i.e. if your big model consists of 5 linear layers, you could make a single, activation-parametric wrapper for 1 layer (including dropouts, layer norms, whatever), and then another wrapper for a flow of N layers, which asks once for which activation function to initialize its children with. In other words, generalize and parameterize when you anticipate this to save you from extra effort and copy-pasting code in the future, but don't overdo it or you'll end up far away from your original specifications and needs.

ps: I don't know whether calling activation functions functors is justifiable.


Instruction: Fast computation for changing the leftmost different bit

Input:
Given the two number in 8-bit:
x = 0b11110111
y = 0b11001010

What I want to do is to compare x and y and change x only the first different leftmost bit based on y. For example:
z = 0b11010111 (Because the leftmost different bit between x and y is in the third place, therefore, change the third bit in x based on y and other remain the same.)

And my code is:
flag = True
for i in range(8):
    if flag and x[i] != y[i]: # Change only the left-most different bit.
        flag = False
    else:
        y[i] = x[i] # Otherwise,  remain the same.

This could work find.
Buit the problem is if I have many pairs like:
for (x, y) in nums:
    flag = True
    for i in range(8):
        if flag and x[i] != y[i]: # Change only the left-most different bit.
            flag = False
        else:
            y[i] = x[i] # Otherwise,  remain the same.

When nums is large, then this process will be really slow.
So how can I improve the process of the problem?
BTW, this is the project of the deep learning task, so it can run on GPU, but I don't know whether it can be paralleled by GPU or not.

Output:
The function you're after:
from math import floor, log2

def my_fun(x, y):
    return x ^ (2 ** floor(log2(x ^ y)))

z = my_fun(0b11110111, 0b11001010)
print(f'{z:b}')

Output:
11010111

The function does the following:

compute the XOR result of x and y, which will include the most significant bit where they differ as the most significant bit to be 1
compute the floor of the log2 of that value, and raising 2 to that power, to get a number that only has that bit set to 1
return the XOR of x and that number, flipping the relevant bit