Instruction: Normalization of the dataset, Error: all elements of input should be between 0 and 1

Input:
I have a problem with data normalization in PyTorch when I try to execute the training. First thing you need to know is that the dataset is composed of 3024 signal windows (so 1 channel), each one with a length of 5000 samples, so the dimension of the CSV file is 5000x3024. Each signal has 1 label that needs to be predicted.
Here is the code for how I load and normalize the data:
class CSVDataset(Dataset):
    # load the dataset
    def __init__(self, path, normalize = False):
        # load the csv file as a dataframe
        df = read_csv(path)
        df = df.transpose()
        # store the inputs and outputs
        self.X = df.values[:, :-1]
        self.y = df.values[:, -1]
        print("Dataset length: ", self.X.shape[0])
        # ensure input data is floats
        self.X = self.X.astype(np.float)
        self.y = self.y.astype(np.float)

        if normalize:
            self.X = self.X.reshape(self.X.shape[1], self.X.shape[0])
            min_X = np.min(self.X,0)  # returns an array of means for each signal window
            max_X = np.max(self.X,0)
            self.X = (self.X - min_X)/(max_X-min_X)
            min_y = np.min(self.y)
            max_y = np.max(self.y)
            self.y = (self.y - min_y)/(max_y-min_y)

        # reshape input data
        self.X = self.X.reshape(self.X.shape[0], 1, self.X.shape[1])
        self.y = self.y.reshape(self.y.shape[0], 1)
        # label encode target and ensure the values are floats
        self.y = LabelEncoder().fit_transform(self.y)
        self.y = self.y.astype(np.float)

# prepare the dataset
def prepare_data(path):
    # load the dataset
    dataset = CSVDataset(path, normalize = True)
    # calculate split
    train, test = dataset.get_splits()
    # prepare data loaders
    train_dl = DataLoader(train, batch_size=32, shuffle=True)
    test_dl = DataLoader(test, batch_size=1024, shuffle=False)
    return train_dl, test_dl

While the train method is:
def train_model(train_dl, model):
    # define the optimization
    criterion = BCELoss()
    optimizer = SGD(model.parameters(), lr=0.01, momentum=0.9)
    model = model.float()
    # enumerate epochs
    for epoch in range(100):
        # enumerate mini batches
        for i, (inputs, targets) in enumerate(iter(train_dl)):
            targets = torch.reshape(targets, (32, 1))
            # clear the gradients
            optimizer.zero_grad()
            # compute the model output
            yhat = model(inputs.float())
            # calculate loss
            loss = criterion(yhat, targets.float())
            # credit assignment
            loss.backward()
            # update model weights
            optimizer.step()

The error that I get is in the line loss = criterion(yhat, targets.float()) and it says:
RuntimeError: all elements of input should be between 0 and 1

I have tried inspecting the X in the variable explorer and it doesn't seem that there are any values that are not between 0 and 1. I don't know what I could have done wrong in normalization. Can you help me?

Output:
Builtin loss functions refer to input and target to designate the prediction and label instances respectively. The error message should be understood as "input of the criterion" i.e. yhat, and not as "input of the model".
It seems yhat does not belong in [0, 1], while BCELoss expects a probability, not a logit. You can either

add a sigmoid layer as the last layer of your model, or

use nn.BCEWithLogitsLoss instead, which combines a sigmoid and the bce loss.


Instruction: Linear autoencoder using Pytorch

Input:
How do we build a simple linear autoencoder and train it using torch.optim optimisers?
How do I do it using autograd (.backward()) and optimising the MSE loss, and then learn the values of the weights and biases in the encoder, and the decoder (ie. 3 parameters in the encoder and 4 in the decoder)? And the data has to be randomized, for each run of learning, start from random weights and biases, such as:
wEncoder = torch.randn(D,1, requires_grad=True)
wDecoder = torch.randn(1,D, requires_grad=True)
bEncoder = torch.randn(1, requires_grad=True)
bDecoder = torch.randn(1,D, requires_grad=True)

The target optimizer is SGD, learning rate 0.01, no momentum, and 1000 steps (from a random start), then how do we plot loss versus epochs (steps)?
I tried this but the losses are the same for every epoch.
D = 2
x = torch.rand(100,D)
x[:,0] = x[:,0] + x[:,1]
x[:,1] = 0.5*x[:,0] + x[:,1]

loss_fn = nn.MSELoss()
optimizer = optim.SGD([x[:,0],x[:,1]], lr=0.01)
losses = []
for epoch in range(1000):
    running_loss = 0.0
    inputs = x_reconstructed
    targets = x
    loss=loss_fn(inputs,targets)
    loss.backward(retain_graph=True)
    optimizer.step()
    optimizer.zero_grad()
    running_loss += loss.item()
    epoch_loss = running_loss / len(data)
    losses.append(running_loss)

Output:
This example should get you going. Please see code comments for further explanation:
import torch

# Use torch.nn.Module to create models
class AutoEncoder(torch.nn.Module):
    def __init__(self, features: int, hidden: int):
        # Necessary in order to log C++ API usage and other internals
        super().__init__()
        self.encoder = torch.nn.Linear(features, hidden)
        self.decoder = torch.nn.Linear(hidden, features)

    def forward(self, X):
        return self.decoder(self.encoder(X))

    def encode(self, X):
        return self.encoder(X)

# Random data
data = torch.rand(100, 4)
model = AutoEncoder(4, 10)
# Pass model.parameters() for increased readability
# Weights of encoder and decoder will be passed
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
loss_fn = torch.nn.MSELoss()

# Per-epoch losses are gathered
# Loss is the mean of batch elements, in our case mean of 100 elements
losses = []
for epoch in range(1000):
    reconstructed = model(data)
    loss = loss_fn(reconstructed, data)
    # No need to retain_graph=True as you are not performing multiple passes
    # of backpropagation
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    losses.append(loss.item())

Please notice linear autoencoder is roughly equivalent to PCA decomposition, which is more efficient.
You should probably use a non-linear autoencoder unless it is simply for training purposes.


Instruction: how to convert a csv file to character level one-hot-encode matrices?

Input:
I have a CSV file that looks like this

I want to choose the last column and make character level one-hot-encode matrices of every sequence, I use this code and it doesn't work
data = pd.read_csv('database.csv', usecols=[4])
alphabet = ['A', 'C', 'D', 'E', 'F', 'G','H', 'I', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y']
charto = dict((c,i) for i,c in enumerate(alphabet))
iint = [charto[char] for char in data]
onehot2 = []
for s in iint:
    lett = [0 for _ in range(len(alphabet))]
    lett[s] = 1
    onehot2.append(lett)

What do you suggest doing for this task?
(by the way, I want to use this dataset for a PyTorch model)

Output:
I think it would be best to keep pd.DataFrame as is and do the transformation "on the fly" within PyTorch Dataset.
First, dummy data similar to yours:
df = pd.DataFrame(
    {
        "ID": [1, 2, 3],
        "Source": ["Serbia", "Poland", "Germany"],
        "Sequence": ["ABCDE", "EBCDA", "AAD"],
    }
)

After that, we can create torch.utils.data.Dataset class (example alphabet is shown, you might change it to anything you want):
class Dataset(torch.utils.data.Dataset):
    def __init__(self, df: pd.DataFrame):
        self.df = df
        # Change alphabet to anything you need
        alphabet = ["A", "B", "C", "D", "E", "F"]
        self.mapping = dict((c, i) for i, c in enumerate(alphabet))

    def __getitem__(self, index):
        sample = df.iloc[index]
        sequence = sample["Sequence"]
        target = torch.nn.functional.one_hot(
            torch.tensor([self.mapping[letter] for letter in sequence]),
            num_classes=len(self.mapping),
        )
        return sample.drop("Sequence"), target

    def __len__(self):
        return len(self.df)

This code simply transforms indices of letters to their one-hot encoding via torch.nn.functional.one_hot function.
Usage is pretty simple:
ds = Dataset(df)
ds[0]

which returns (you might want to change how your sample is created though as I'm not sure about the format and only focused on hot-encoded targets) the following targets (ID and Source omitted):
tensor([ [1., 0., 0., 0., 0., 0.],
         [0., 1., 0., 0., 0., 0.],
         [0., 0., 1., 0., 0., 0.],
         [0., 0., 0., 1., 0., 0.],
         [0., 0., 0., 0., 1., 0.]]))


Instruction: Why does Keras BatchNorm produce different output than PyTorch?

Input:
Torch：'1.9.0+cu111'
Tensorflow-gpu：'2.5.0'
I came across a strange thing, when using the Batch Normal layer of tensorflow 2.5 and the BatchNorm2d layer of Pytorch 1.9 to calculate the same Tensor , and the results were quite different (TensorFlow is close to 1, Pytorch is close to 0).I thought at first it was the difference between the momentum and epsilon , but after changing them to the same, the result was the same.
from torch import nn
import torch
x = torch.ones((20, 100, 35, 45))
a = nn.Sequential(
            # nn.Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), padding=0, bias=True),
            nn.BatchNorm2d(100)
        )
b = a(x)

import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.layers import *
x = tf.ones((20, 35, 45, 100))
a = keras.models.Sequential([
            # Conv2D(128, (1, 1), (1, 1), padding='same', use_bias=True),
            BatchNormalization()
        ])
b = a(x)

Output:
Batchnormalization works differently in training and inference,
During training (i.e. when using fit() or when calling the layer/model with the argument training=True), the layer normalizes its output using the mean and standard deviation of the current batch of inputs. That is to say, for each channel being normalized, the layer returns
gamma * (batch - mean(batch)) / sqrt(var(batch) + epsilon) + beta

where:

epsilon is small constant (configurable as part of the constructor arguments)
gamma is a learned scaling factor (initialized as 1), which can be disabled by passing scale=False to the constructor.
beta is a learned offset factor (initialized as 0), which can be disabled by passing center=False to the constructor.

During inference (i.e. when using evaluate() or predict() or when calling the layer/model with the argument training=False (which is the default), the layer normalizes its output using a moving average of the mean and standard deviation of the batches it has seen during training. That is to say, it returns
gamma * (batch - self.moving_mean) / sqrt(self.moving_var + epsilon) + beta.

self.moving_mean and self.moving_var are non-trainable variables that are updated each time the layer in called in training mode, as such:
    moving_mean = moving_mean * momentum + mean(batch) * (1 - momentum)
    moving_var = moving_var * momentum + var(batch) * (1 - momentum)

ref: https://www.tensorflow.org/api_docs/python/tf/keras/layers/BatchNormalization
If you run the pytorch batchnorm in eval mode, you get close results (the rest of the discrepancy comes from the different internal implementation, parameter choices, etc.),
from torch import nn
import torch
x = torch.ones((1, 2, 2, 2))
a = nn.Sequential(
            # nn.Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), padding=0, bias=True),
            nn.BatchNorm2d(2)
        )
a.eval()
b = a(x)
print(b)
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.layers import *
x = tf.ones((1, 2, 2, 2))
a = keras.models.Sequential([
            # Conv2D(128, (1, 1), (1, 1), padding='same', use_bias=True),
            BatchNormalization()
        ])
b = a(x)
print(b)

out:
tensor([[[[1.0000, 1.0000],
          [1.0000, 1.0000]],

         [[1.0000, 1.0000],
          [1.0000, 1.0000]]]], grad_fn=<NativeBatchNormBackward>)
tf.Tensor(
[[[[0.9995004 0.9995004]
   [0.9995004 0.9995004]]

  [[0.9995004 0.9995004]
   [0.9995004 0.9995004]]]], shape=(1, 2, 2, 2), dtype=float32)


Instruction: batch_first in PyTorch LSTM

Input:
I am new in this field, so I still don't understand about the batch_first in PyTorch LSTM. I tried the code that someone has referred to me, and it works on my train data when the batch_first = False, it produces the same output for Official LSTM and Manual LSTM. However, when I change the batch_first = True, it not produces the same value anymore, while I need to change the batch_first to True, because my dataset shape is tensor of (Batch, Sequences, Input size). Which part of the Manual LSTM needs to be changed to produces the same output as the Official LSTM produces when batch_first = True? Here is the code snippet:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

train_x = torch.tensor([[[0.14285755], [0], [0.04761982], [0.04761982], [0.04761982],
  [0.04761982], [0.04761982], [0.09523869], [0.09523869], [0.09523869],
  [0.09523869], [0.09523869], [0.04761982], [0.04761982], [0.04761982],
  [0.04761982], [0.09523869], [0.        ], [0.        ], [0.        ],
  [0.        ], [0.09523869], [0.09523869], [0.09523869], [0.09523869],
  [0.09523869], [0.09523869], [0.09523869],[0.14285755], [0.14285755]]],
  requires_grad=True)

seed = 23
torch.manual_seed(seed)
np.random.seed(seed)

pytorch_lstm = torch.nn.LSTM(1, 1, bidirectional=False, num_layers=1, batch_first=True)
weights = torch.randn(pytorch_lstm.weight_ih_l0.shape,dtype = torch.float)
pytorch_lstm.weight_ih_l0 = torch.nn.Parameter(weights)
# Set bias to Zero
pytorch_lstm.bias_ih_l0 = torch.nn.Parameter(torch.zeros(pytorch_lstm.bias_ih_l0.shape))
pytorch_lstm.weight_hh_l0 = torch.nn.Parameter(torch.ones(pytorch_lstm.weight_hh_l0.shape))
# Set bias to Zero
pytorch_lstm.bias_hh_l0 = torch.nn.Parameter(torch.zeros(pytorch_lstm.bias_ih_l0.shape))
pytorch_lstm_out = pytorch_lstm(train_x)

batch_size=1

# Manual Calculation
W_ii, W_if, W_ig, W_io = pytorch_lstm.weight_ih_l0.split(1, dim=0)
b_ii, b_if, b_ig, b_io = pytorch_lstm.bias_ih_l0.split(1, dim=0)

W_hi, W_hf, W_hg, W_ho = pytorch_lstm.weight_hh_l0.split(1, dim=0)
b_hi, b_hf, b_hg, b_ho = pytorch_lstm.bias_hh_l0.split(1, dim=0)

prev_h = torch.zeros((batchsize,1))
prev_c = torch.zeros((batchsize,1))

i_t = torch.sigmoid(F.linear(train_x, W_ii, b_ii) + F.linear(prev_h, W_hi, b_hi))
f_t = torch.sigmoid(F.linear(train_x, W_if, b_if) + F.linear(prev_h, W_hf, b_hf))
g_t = torch.tanh(F.linear(train_x, W_ig, b_ig) + F.linear(prev_h, W_hg, b_hg))
o_t = torch.sigmoid(F.linear(train_x, W_io, b_io) + F.linear(prev_h, W_ho, b_ho))
c_t = f_t * prev_c + i_t * g_t
h_t = o_t * torch.tanh(c_t)

print('nn.LSTM output {}, manual output {}'.format(pytorch_lstm_out[0], h_t))
print('nn.LSTM hidden {}, manual hidden {}'.format(pytorch_lstm_out[1][0], h_t))
print('nn.LSTM state {}, manual state {}'.format(pytorch_lstm_out[1][1], c_t))

Output:
You have to iterate through each sequence element at a time and take the computed hidden and cell states as input in the next time step...
h_t = torch.zeros((batch_size,1))
c_t = torch.zeros((batch_size,1))

hidden_seq = []

for t in range(30):
  x_t = train_x[:, t, :]
  i_t = torch.sigmoid(F.linear(x_t, W_ii, b_ii) + F.linear(h_t, W_hi, b_hi))
  f_t = torch.sigmoid(F.linear(x_t, W_if, b_if) + F.linear(h_t, W_hf, b_hf))
  g_t = torch.tanh(F.linear(x_t, W_ig, b_ig) + F.linear(h_t, W_hg, b_hg))
  o_t = torch.sigmoid(F.linear(x_t, W_io, b_io) + F.linear(h_t, W_ho, b_ho))
  c_t = f_t * c_t + i_t * g_t
  h_t = o_t * torch.tanh(c_t)
  hidden_seq.append(h_t.unsqueeze(0))

hidden_seq = torch.cat(hidden_seq, dim=0)
hidden_seq = hidden_seq.transpose(0, 1).contiguous()

print('nn.LSTM output {}, manual output {}'.format(pytorch_lstm_out[0], hidden_seq))


Instruction: Wrong "-1 background" annotations loaded from Custom COCO Dataset using Mmdetection

Input:
Introduction
I'm working using Mmdetection to train a Deformable DETR model using a custom COCO Dataset. Meaning a Custom Dataset using the COCO format of annotations. The dataset uses the same images as the COCO with different "toy" annotations for a "playground" experiment and the annotation file was created using the packages pycocotools and json exclusively.
I have made five variations of this playground dataset: 2 datasets with three classes (classes 1, 2, and 3), 1 dataset with six classes (classes 1 to 6) and 2 datasets with 7 classes (classes 1 to 7).
The Problem
Now, after creating the dataset in mmdetection using mmdet.datasets.build_dataset, I used the following code to check if everything was OK:
from pycocotools.coco import COCO
from os import path as osp
from mmdet.datasets import build_dataset

cfg = start_config() # this is simply a function to startup the config file
ann_file = osp.join(cfg.data.train.data_root, cfg.data.train.ann_file)
coco = COCO(ann_file)
img_ids = coco.getImgIds()
ann_ids = coco.getAnnIds(imgIds=img_ids)
anns = coco.loadAnns(ids=ann_ids)

cats_counter = {}
for ann in anns:
  if ann['category_id'] in cats_counter:
    cats_counter[ann['category_id']]+=1
  else:
    cats_counter[ann['category_id']] = 1
print(cats_counter)

cats = {cat['id']:cat for cat in coco.loadCats(coco.getCatIds())}
for i in range(len(cats_counter)):
  print("{} ({}) \t|\t{}".format(i, cats[i]['name'], cats_counter[i]))

ds = build_dataset(cfg.data.train)
print(ds)

For three of the datasets the amounts from the json file and from the constructed mmdet dataset are almost exactly equal. However, for one of the 3-classes dataset and for the 6-classes dataset, the results are incredibly different, where this code returns the following:
{3: 1843, 1: 659, 4: 1594, 2: 582, 0: 1421, 5: 498}
0 (1)   |   1421
1 (2)   |   659
2 (3)   |   582
3 (4)   |   1843
4 (5)   |   1594
5 (6)   |   498
loading annotations into memory...
Done (t=0.06s)
creating index...
index created!

CocoDataset Train dataset with number of images 1001, and instance counts:
+---------------+-------+---------------+-------+---------------+-------+---------------+-------+---------------+-------+
| category      | count | category      | count | category      | count | category      | count | category      | count |
+---------------+-------+---------------+-------+---------------+-------+---------------+-------+---------------+-------+
| 0 [1]         | 1421  | 1 [2]         | 659   | 2 [3]         | 581   | 3 [4]         | 1843  | 4 [5]         | 1594  |
|               |       |               |       |               |       |               |       |               |       |
| 5 [6]         | 0     | -1 background | 45    |               |       |               |       |               |       |
+---------------+-------+---------------+-------+---------------+-------+---------------+-------+---------------+-------+

and
{1: 1420, 0: 4131, 2: 1046}
0 (1)   |   4131
1 (2)   |   1420
2 (3)   |   1046
loading annotations into memory...
Done (t=0.06s)
creating index...
index created!

CocoDataset Train dataset with number of images 1001, and instance counts:
+----------+-------+------------+-------+----------+-------+---------------+-------+----------+-------+
| category | count | category   | count | category | count | category      | count | category | count |
+----------+-------+------------+-------+----------+-------+---------------+-------+----------+-------+
|          |       |            |       |          |       |               |       |          |       |
| 0 [1]    | 1419  | 1 [2]      | 0     | 2 [3]    | 0     | -1 background | 443   |          |       |
+----------+-------+------------+-------+----------+-------+---------------+-------+----------+-------+

You can see that there is no "-1" id in the annotation json, and also some of the classes from the 3-classes dataset have 0 annotations, while the json clearly shows more than that. Has anyone encountered something similar using Mmdetection? What could be causing this problem?

Output: There was a mismatch between the classes names in the annotation file and the classes names in the mmdetection config object. Correcting those solved the problem.


Instruction: Why does PyTorch's built-in loss function only work with the Long tensor type?

Input: So, I'm using: torch.nn.CrossEntropyLoss(predictions, targets), and I wonder why exactly targets need to be a 64-bit integer and not 32-bit?

Output:
I wonder why exactly targets need to be a 64-bit integer and not 32-bit?

This is because PyTorch is precompiled. Some other frameworks that preceded it (not Tensorflow) would invoke the compiler on-the-fly, which can cause a delay and some other unpleasantness. PyTorch doesn't. But this means the developers have to be mindful of the size of the precompiled library. They have to balance the utility of supporting yet another data type vs the increase in size that compiling everything for that data type would cause, and the decision here went against int32, uint32, uint64, etc.


Instruction: Why do you multiply the two images to see the correlation?

Input:
I have some questions about the CP Viton module:
feature_A = feature_A.transpose(2,3).contiguous().view(b,c,h*w)

feature_B = feature_B.view(b,c,h*w).transpose(1,2)

# perform matrix mult.
feature_mul = torch.bmm(feature_B,feature_A)
print(feature_mul.size()) #torch.Size([4, 192, 192])

at this code

For the multiplication of matrices, I don't know why they make it like b,hw,hw.

It is said that multiplying the shape of the image as follows is to extract the correlation, but I don't know why. I'm talking about the bmm part.

Output: The spatial correlation consists of computing the dot product of feature vectors of every (feature_A[k,:,i], feature_B[k,:,j]) feature pair. As such you first need to flatten the spatial dimension which results in a dimension of size h*w on both tensors. Your two operands will have a shape of (b, c, hw), and (b, hw, c). As a result of applying bmm, you end up with a tensor shaped (b, hw, hw)


Instruction: How to update a pretrained model after Pruning of filters in its conv layer in PyTorch?

Input:
I have a pretrained model LeNet5 defined from scratch. I am performing pruning over filters in the convolution layers present in the model shown below.
class LeNet5(nn.Module):

    def __init__(self, n_classes):
        super(LeNet5, self).__init__()
        self.feature_extractor = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=20, kernel_size=5, stride=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
            nn.Conv2d(in_channels=20, out_channels=50, kernel_size=5, stride=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2)
        )
        self.classifier = nn.Sequential(
            nn.Linear(in_features=800, out_features=500),
            nn.ReLU(),
            nn.Linear(in_features=500, out_features=10), # 10 - possible classes
        )

    def forward(self, x):
        #x = x.view(x.size(0), -1)
        x = self.feature_extractor(x)
        x = torch.flatten(x, 1)
        logits = self.classifier(x)
        probs = F.softmax(logits, dim=1)
        return logits, probs

I have successfully removed 2 filters from 20 in layer 1 (now 18 filters in conv2d layer1) and 5 filters from 50 in layer 2 (now 45 filters in conv2d layer3). So, now I need to update the model with the changes done as follows -

out_channel of layer 1 - 20 to 18
in_channel of layer 3 - 20 to 18
out_channel of layer 3 - 50 to 45

However, I'm unable to run the model as it gives dimension error.
RuntimeError: mat1 and mat2 shapes cannot be multiplied (32x720 and 800x500)

How to update the no. of filters layers present in the model using Pytorch to perform pruning? Is there any library I can use for the same?

Output:
Assuming you do not want the model to automatically change structure during runtime, you can easily update the structure of the model by simply changing the input parameters to the constructor. For instance:
nn.Conv2d(in_channels = 1, out_channels = 18, kernel_size = 5, stride = 1),
nn.Conv2d(in_channels = 18, out_channels = 45, kernel_size = 5, stride = 1),

and so on.
If you are retraining from scratch every time you change the model structure, that's all you need to do. However, if you would like to maintain portions of the already learned parameters when you change the model, you'll need to select these relevant values and reassign them to the model parameters. For instance, consider the parameters associated with the first convolutional layer, 1 input, 20 outputs, and kernel size of 5. The weights and biases for this layer have size [1,20,5,5] and [1,20]. You need to modify these parameters such that they have size [1,18,5,5] and [1,18]. You'd thus need the indices for the particular kernels/filters you want to maintain and which kernels you'd like to prune. The code syntax for doing this is roughly:
params = net.state_dict()
params["feature_extractor"]["conv1.weight"] = params["feature_extractor"]["conv1.weight"][:,:18,:,:]
params["feature_extractor"]["conv1.bias"] = params["feature_extractor"]["conv1.bias"][:,:18]
# and so on for the other layers

net.load_state_dict(params)

Here, I simply drop the last two kernels/bias values for the first convolutional layer. (Note that the actual dictionary key names may differ slightly; I didn't code this up to check because, as indicated in the comments above, you included a picture of code rather than real, copy-able, code so try to do the latter in the future.)


Instruction: Create tensor with arrays of different dimensions in PyTorch

Input:
I want to concatenate arrays of different dimensions to feed them to my neural network that will have as first layer the AdaptiveAveragePooling1d. I have a dataset that is composed of several signals (1D arrays), each one with a different length. For example:
array1 = np.random.randn(1200,1)
array2 = np.random.randn(950,1)
array3 = np.random.randn(1000,1)

I want to obtain a tensor in which I concatenate these three signals to obtain a 2D tensor.
However if I try to do
tensor = torch.Tensor([array1, array2, array3])

It gives me this error:
ValueError: expected sequence of length 1200 at dim 2 (got 950)

Is there a way to obtain such thing?
EDIT
More information about the dataset:

Each signal window represents a heart beat on the ECG registration, taken from several patients, sampled with a sampling frequency of 1000Hz
The beats can have different lengths, because it depends on the heart rate of the patient itself
For each beat I need to predict the length of the QRS interval (the target of the network) that I have, expressed in milliseconds
I have already thought of interpolating the shortest samples to the the length of the longest ones, but then I would also have to change the length of the QRS interval in the labels, is that right?

I have read of this AdaptiveAveragePooling1d layer, that would allow me to input the network with samples of different sizes. But my problem is how do I input the network a dataset in which each sample has a different length? How do I group them without using a filling method with NaNs or zeros?
I hope I explained myself.

Output:
This disobeys the definition of a tensor and is impossible. If a tensor is of shape (NxMx1), all of the N matrices must be of size (Mx1).
There are still ways to get all your arrays to the same length. Look at where your data is coming from and what its structure is and figure out which of the following solutions would work. Some of these may change the signal's derivative in a way you don't like

Cropping arrays to the same size (ie cutting start/end off) or zero padding the shorter ones to the length of the longer one (I really dislike this one and it would only work for very specific applications)
'Stretching' the arrays to the same size by using interpolation
Shortening the arrays to the same size by subsampling
For some applications, maybe even passing the coefficients of a
fourier series from the signals

EDIT
For heart rate, which should be a roughly periodic signal, I'd definitely crop the signal which should work quite well. Passing FFT(equally cropped signals) or Fourier coefficients may also yield interesting results, but from my experience with neural spike data, training on the FFT of a signal like this doesn't perform any better when you have enough data to train off.
Also if you're using a fully connected network, a using 1D convolutions is a good alternative to try.


Instruction: Initializing model parameters in pytorch manually

Input:
I am creating a separate class to initializer model and adding layers in a list, but those layers are not being added to parameters, plz tell how to add them to parameters() of model.
class Mnist_Net(nn.Module):
def __init__(self,input_dim,output_dim,hidden_layers=2,neurons=128):
    super().__init__()
    layers = []
    for i in range(hidden_layers):
        if len(layers) == 0:
            layers.append(nn.Linear(input_dim,neurons))
        if i == hidden_layers-1:
            layers.append(nn.Linear(layers[-2].weight.shape[0],output_dim))
        layers.append(nn.Linear(layers[i-1].weight.shape[0],neurons))
    self.layers= layers

When I print model.parameters()
model = Mnist_Net(28*28,10,neurons=56)
   for t in model.parameters():
   print(t)

it shows nothing, but when I add layers in class like
self.layer1 = nn.Linear(input_dim,neurons)

It shows one layer in parameters.Plz tell How can I add all layers in self.layers in model.parameters()

Output:
To be registered in the parent module, your submodules should be nn.Modules themselves. In your case, you should wrap layers with nn.ModuleList:
  self.layers = nn.ModuleList(layers)

Then, your layers will be registered:
>>> model = Mnist_Net(28*28,10, neurons=56)

>>> for t in model.parameters():
...    print(t.shape)
torch.Size([56, 784])
torch.Size([56])
torch.Size([56, 56])
torch.Size([56])
torch.Size([10, 56])
torch.Size([10])
torch.Size([56, 56])
torch.Size([56])


Instruction: Spacy inference goes OOM when processing several documents

Input:
I'm using spacy to process documents that come through rest api. To be more specific, I'm using transformer based model en_core_web_trf for NER, running on GPU. Here is a code snippet of the spacy related class (It is packed inside some basic flask server and but I don't suppose that matters here)
class SpacyExtractor():
    def __init__(self):
        spacy.require_gpu()
        self.model = spacy.load('en_core_web_trf',
                                 disable=["tagger", "parser", "attribute_ruler", "lemmatizer"])

    def get_named_entities(self, text: str):
        doc = self.model(text)
        entities = []
        for ent in doc.ents:
            entities.append((ent.text, ent.label_))
        return entities

The problem is, with each call of get_named_entities, the amount of GPU memory allocated goes up. And it is like 2-3 GB every time (I checked this by repeatedly calling nvidia-smi while the app was processing the docs). So after a few calls, I get OOM error
RuntimeError: CUDA out of memory. Tried to allocate 2.35 GiB (GPU 0; 10.76 GiB total capacity; 5.02 GiB already allocated; 1.18 GiB free; 8.41 GiB reserved in total by PyTorch)
Documents are not huge at all, 1-100 pages of text for each one.
I think I make some mistake, but I just don't see it.
Environment: Ubuntu 18.04, Python 3.8, spacy 3.1.3, cuda 9.1, RTX 2080Ti 11GB RAM
EDIT: Also, I found out the OOM error when processing a single really long document, presented as a single long string.

Output:
The problem is, with each call of get_named_entities, the amount of GPU memory allocated goes up.

You should detach your data as explained in the FAQ:

Don’t accumulate history across your training loop. By default,
computations involving variables that require gradients will keep
history. This means that you should avoid using such variables in
computations which will live beyond your training loops, e.g., when
tracking statistics. Instead, you should detach the variable or access
its underlying data.

Edit
You can also use
with torch.no_grad():
    doc = self.model(text)

EDIT: Also, I found out the OOM error when processing a single really long document, presented as a single long string.

Well, this is to be expected.


Instruction: Fastest method of reading tensor objects from files in python

Input:
I am training PyTorch models on various datasets. The datasets up to this point have been images so I can just read them on the fly when needed using cv2 or PIL which is fast.
Now I am presented with a dataset of tensor objects of shape [400, 400, 8]. In the past I have tried to load these objects using PyTorch and NumPy's built-in tensor reading operations but these are generally much slower than reading images.
The objects are currently stored in h5py compressed files where there are ~800 per file. My plan was to save the objects individually in some format and then read them on the fly but I am unsure of what format to save them in which is fastest.
I would like to avoid keeping them all in memory as I believe the memory requirement would be too high.

Output:
If the data arrays are still "images", just 8-channel ones, you can split them into 3 image files
a = x[:, :, 0:3]
b = x[:, :, 3:6]
c = x[:, :, 5:8]
c[:, :, 0] = 0 # reduces the compressed size

and store them using the conventional image libraries (cv2 and PIL).
Images compress much better than general data (lossy 'jpeg' even more so), and thefore that reduces both the disk space and bandwidth, and has file system caching benefits.


Instruction: pytorch model predicts fixed label when it exports to onnx

Input:
I trained resnet-18 model in pytorch. And it works well in pytorch.
But, when I converts it to onnx and predicts in cv2, model predicts only 1~2 label(it should predict 0~17 labels).
this is my model export code
    model.eval()
    x = torch.randn(1, 3, 512, 384, requires_grad=True)

    # export model
    torch.onnx.export(model, x, "model.onnx", export_params=True, opset_version=10, do_constant_folding=True, input_names = ['input'], output_names = ['output'])

And this is my code for inference in cv2
self.transform = albumentations.Compose([
        albumentations.Resize(512, 384, cv2.INTER_LINEAR),
        albumentations.GaussianBlur(3, sigma_limit=(0.1, 2)),
        albumentations.Normalize(mean=(0.5), std=(0.2)),
        albumentations.ToFloat(max_value=255)
        ])
...
#image crop code: works fine in pytorch
image = frame[ymin:ymax, xmin:xmax]  #type(frame)==numpy.array, RGB form
augmented = self.transform(image=image)
image = augmented["image"]
...
#inference code: does not work well
net=cv2.dnn.readNet("Model.onnx")
blob = cv2.dnn.blobFromImage(image, swapRB=False, crop=False)
net.setInput(blob)
label = np.array(net.forward())
text  =  'Label: '+str(np.argmax(label[0]))

All transform settings works well in pytorch.
What can be the problem in this code?

Output: The problem with your code probably has to do with preprocessing the images differently: self.transform rescales the image, but when you are reading blob, you are not doing that. To verify this, you can read the same image and check if the image and blob are equal (e.g. using torch.allclose), when the actual (random) augmentations are disabled.


Instruction: Using CUDA 11.x but getting error: Unknown CUDA arch (8.6) or GPU not supported

Input:
I'm setting up a conda environment to use pytorch 1.4.0 (on Ubuntu 20.04.2), but getting the error message:

ValueError: Unknown CUDA arch (8.6) or GPU not supported

I know this has been asked before, but no answer fits my case. This answer suggests that the CUDA version is too old. However, I updated my CUDA version to the most recent, and get the same error message.
nvcc -V says I have CUDA 11 installed, and when I run nvidia-smi I get this info:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.84       Driver Version: 460.84       CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+

which, according to the NVIDIA docs, should work be compatible:

Another auxilliary question: What does the "8.6" in CUDA arch (8.6) represent?

Output:
Specific versions of PyTorch work only with specific versions of CUDA.
If you are using CUDA-11.1, you'll need a fairly recent version of PyTorch. You need to either upgrade your PyTorch, or downgrade your CUDA.


Instruction: Getting RESNet18 to work with float32 data

Input:
I have float32 data that I am trying to get RESNet18 to work with.  I am using the RESNet model in torchvision (and using pytorch lightning) and modified it to use one layer (grayscale) data like so:
class ResNetMSTAR(pl.LightningModule):
def __init__(self):
  super().__init__()
  # define model and loss
  self.model = resnet18(num_classes=3)
  self.model.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  self.loss = nn.CrossEntropyLoss()

@auto_move_data # this decorator automatically handles moving your tensors to GPU if required
def forward(self, x):
return self.model(x)

def training_step(self, batch, batch_no):
  # implement single training step
  x, y = batch
  logits = self(x)
  loss = self.loss(logits, y)
  return loss

def configure_optimizers(self):
  # choose your optimizer
  return torch.optim.RMSprop(self.parameters(), lr=0.005)

When I try to run this model I am getting the following error:
File "/usr/local/lib64/python3.6/site-packages/torch/nn/functional.py", line 2824, in cross_entropy
    return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss_forward

Is there anything that I can do differently to keep this error from happening?

Output:
The problem is that the y your feeding your cross entropy loss, is not a LongTensor, but a FloatTensor. CrossEntropy expects getting fed a LongTensor for the target, and raises the error.
This is an ugly fix:
x, y = batch
y = y.long()

But what I recommend you to do is going to where the dataset is defined, and make sure you are generating long targets, this way you won't reproduce this error if you change how your training loop is working.


Instruction: PyTorch Generating Matrix using a Kernel Function without For-Loops

Input:
I am trying to generate a matrix (tensor object on PyTorch) that is similar to Gram matrix except I need to apply a kernel function instead of inner product on my input matrix.
For loops like the one below works:
N = x.shape[0] # x.shape = (N,d)
G = torch.zeros((N,N))
for i in range(N):
    for j in range(N):
        G[i][j] = K(x[i], x[j])

where x is my input tensor whose shape is (N,d) and the kernel function K(a,b) yields a real value after performing some math. For example:
def K(a,b):
   return ((1+(a*b)).sum()).pow(2) #second degree polynomial.

I want to generate this matrix, G without having to change the kernel function K() and of course, without for-loops!
My initial attempt is to use a lambda approach but this code below obviously doesn't work as it only yields a list of k(x[i],x[i]).
G = torch.tensor(list(map(lambda a,b: K(a,b),x,x))

How can I use the lambda function to yield N-by-N matrix?
What would be some other ways to tackle this problem?
Any insight would be appreciated.

Output:
You can calculate G from x simply with:
G = (1 + torch.matmul(x, x.T)).pow(2)


Instruction: Synthesizing 1x1 convolution layer with fully connected layers

Input:
I'm trying to synthesize a 1x1 convolution layer with fully connected layers. This means a fully connected neural network deciding the parameters of a 1x1 convolution layer. Here is how I do.
class Network(nn.Module):
def __init__(self, len_input, num_kernels):
    self.input_layers = nn.Sequential(
        nn.Linear(len_input, num_kernels * 2),
        nn.ReLU(),
        nn.Linear(num_kernels * 2, num_kernels),
        nn.ReLU()
    )

    self.synthesized_conv = nn.Conv2d(in_channels=3, out_channels=num_kernels, bias=False, kernel_size=1)

    self.conv_layers = nn.Sequential(
        nn.ReLU(),
        nn.Conv2d(in_channels=num_kernels, out_channels=3, kernel_size=1)
    )

def forward(self, x1, img):
    x = self.input_layer(x1.float())
    with torch.no_grad():
        self.synthesized_conv.weight = nn.Parameter(x.reshape_as(self.synthesized_conv.weight))
    generated = self.conv_layer(self.synthesized_conv(img))
    return generated

There you can see that I'm initializing a 1x1 conv layer called "synthesized_conv" and trying to replace it's parameters with a fully connected network output called "self.input_layers" with call-by-reference. However, gradients doesn't seem like flowing through the fully connected network, but only flowing through convolutional layers. Here's how parameter histogram for fully connected layers looks like:

This histogram comes as a strong indicator of those fully connected part is not learning at all. It's most likely a malpractice of convolution parameter update by fully connected network output. Can someone help me how can I do it without breaking the autograd graph?

Output:
The issue is you are redefining, again and again, the weight attribute of your model. An more direct solution would be to use the functional approach, i.e. torch.nn.functional.conv2d:
class Network(nn.Module):
  def __init__(self, len_input, num_kernels):
    super().__init__()
    self.input_layers = nn.Sequential(
        nn.Linear(len_input, num_kernels * 2),
        nn.ReLU(),
        nn.Linear(num_kernels * 2, num_kernels * 3),
        nn.ReLU())

    self.synthesized_conv = nn.Conv2d(
        in_channels=3, out_channels=num_kernels, kernel_size=1)

    self.conv_layers = nn.Sequential(
        nn.ReLU(),
        nn.Conv2d(in_channels=num_kernels, out_channels=3, kernel_size=1))

  def forward(self, x1, img):
    x = self.input_layers(x1.float())
    w = x.reshape_as(self.synthesized_conv.weight)
    generated = F.conv2d(img, w)
    return generated

Also, I believe your input_layers will have to output num_kernels * 3 components in total since you have three channels total on your synthesized convolution.
Here is a test example:
>>> model = Network(10,3)
>>> out = model(torch.rand(1,10), torch.rand(1,3,16,16))
>>> out.shape
(torch.Size([1, 3, 16, 16]), <ThnnConv2DBackward at 0x7fe5d8e41450>)

Of course, the parameters of synthesized_conv will never be changed, since they are never being used to infer the output. You can remove self.synthesized_conv altogether:
class Network(nn.Module):
  def __init__(self, len_input, num_kernels):
    super().__init__()
    self.input_layers = nn.Sequential(
        nn.Linear(len_input, num_kernels * 2),
        nn.ReLU(),
        nn.Linear(num_kernels * 2, num_kernels*3),
        nn.ReLU())

    self.syn_conv_shape = (num_kernels, 3, 1, 1)

    self.conv_layers = nn.Sequential(
        nn.ReLU(),
        nn.Conv2d(in_channels=num_kernels, out_channels=3, kernel_size=1))

  def forward(self, x1, img):
    x = self.input_layers(x1.float())
    generated = F.conv2d(img, x.reshape(self.syn_conv_shape))
    return generated


Instruction: Why doesn't torch pruning actually remove filters or weights?

Input:
I work with one architecture and trying to sparse it via prune. I wrote functions for pruning, here is one of them:
def prune_model_l1_unstructured(model, layer_type, proportion):
    for module in model.modules():
        if isinstance(module, layer_type):
            prune.l1_unstructured(module, 'weight', proportion)
            prune.remove(module, 'weight')
    return model

# prune model
prune_model_l1_unstructured(model, nn.Conv2d, 0.5)

It prunes some weights (change them to zeros). But prune.remove just deletes original weights and keeps zeros instead. Total amount of parameters still same (I checked it). The model's file (model.pt) size still the same too. And the model's "speed" still the same after it. I tried also global pruning and structured L1 pruning, results are the same. So how this can help to improve model's performance time? Why aren't the weights being removed and how to remove pruned connections?

Output:
TLDR; PyTorch prune's function just works as a weight mask, that's all it does. There are no memory savings associated with using torch.nn.utils.prune.

As the documentation page for torch.nn.utils.prune.remove states:

Removes the pruning reparameterization from a module and the pruning method from the forward hook.

In effect, this means it will remove the mask - that prune.l1_unstructured added - from the parameter. As a side effect, removing the prune will imply having zeros on the previously masked values but these won't stay as 0s. In the end, PyTorch prune will only take more memory compared to not using it. So this is not actually the functionality you are looking for.
You can read more on this comment.

Here is an example:
>>> module = nn.Linear(10,3)
>>> prune.l1_unstructured(module, name='weight', amount=0.3)

The weight parameters are masked:
>>> module.weight
tensor([[-0.0000,  0.0000, -0.1397, -0.0942,  0.0000,  0.0000,  0.0000, -0.1452,
          0.0401,  0.1098],
        [ 0.2909, -0.0000,  0.2871,  0.1725,  0.0000,  0.0587,  0.0795, -0.1253,
          0.0764, -0.2569],
        [ 0.0000, -0.3054, -0.2722,  0.2414,  0.1737, -0.0000, -0.2825,  0.0685,
          0.1616,  0.1095]], grad_fn=<MulBackward0>)

Here is the mask:
>>> module.weight_mask
tensor([[0., 0., 1., 1., 0., 0., 0., 1., 1., 1.],
        [1., 0., 1., 1., 0., 1., 1., 1., 1., 1.],
        [0., 1., 1., 1., 1., 0., 1., 1., 1., 1.]])

Notice that when applying prune.remove, the pruning is removed. And, the masked values remain at zero but are "unfrozen":
>>> prune.remove(module, 'weight')

>>> module.weight
tensor([[-0.0000,  0.0000, -0.1397, -0.0942,  0.0000,  0.0000,  0.0000, -0.1452,
          0.0401,  0.1098],
        [ 0.2909, -0.0000,  0.2871,  0.1725,  0.0000,  0.0587,  0.0795, -0.1253,
          0.0764, -0.2569],
        [ 0.0000, -0.3054, -0.2722,  0.2414,  0.1737, -0.0000, -0.2825,  0.0685,
          0.1616,  0.1095]], grad_fn=<MulBackward0>)

And the mask is gone:
>>> hasattr(module, 'weight_mask')
False


Instruction: How to do a masked mean in PyTorch?

Input:
This is the forward pass of a bidirectional rnn where I want to take the avg pool of the output features. As you can see, I'm trying to exclude the time steps with a pad token from the calculation.
def forward(self, text):
    # text is shape (B, L)
    embed = self.embed(text)
    rnn_out, _ = self.rnn(embed)  # (B, L, 2*H)
    # Calculate average ignoring the pad token
    with torch.no_grad():
        rnn_out[text == self.pad_token] *= 0
        denom = torch.sum(text != self.pad_token, -1, keepdim=True)
    feat = torch.sum(rnn_out, dim=1) / denom
    feat =  self.dropout(feat)
    return feat

Backpropagation raises an exception because of the line rnn_out[text == self.pad_token] *= 0. Here's what it looks like:
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [32, 21, 128]], which is output 0 of CudnnRnnBackward, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

What's the correct way to do this?
Note: I know I can do this by doing and/or of the following:

Provide text lengths as an input.
Loop through batch dimension, finding the mean for each sequence then stack the result.

But I want to know if there's a cleaner way not involving those.

Output:
You're modifying a vector in a context where you disable the building of a computational graph (and you modify it inplace using *=), this will wreak havoc on the computation of the gradient. Instead I'd suggest the following:
mask = text != self.pad_token
denom = torch.sum(mask, -1, keepdim=True)
feat = torch.sum(rnn_out * mask.unsqueeze(-1), dim=1) / denom

Maybe you have to tweak this snippet a little bit, I couldn't test it as you haven't provided a complete example, but it hopefully shows the technique you can use.


Instruction: PyTorch - discard dataloader batch

Input:
I have a custom Dataset that loads data from large files. Sometimes, the loaded data are empty and I don't want to use them for training.
In Dataset I have:
   def __getitem__(self, i):
       (x, y) = self.getData(i) #getData loads data and handles problems
       return (x, y)

which in case of bad data return (None, None) (x and y are both None). However, it later fails in DataLoader and I am not able to skip this batch entirely. I have the batch size set to 1.
trainLoader = DataLoader(trainDataset, batch_size=1, shuffle=False)
for x_batch, y_batch in trainLoader:
    #process and train

Output:
You could implement a custom IterableDataset and define a __next__ and __iter__ that would skip any instances for which your getData function has raised an error on:
Here is a possible implementation with dummy data:
class DS(IterableDataset):
    def __init__(self):
        self.data = torch.randint(0,3,(20,))
        self._i = -1

    def getData(self, index):
        x = self.data[index]
        if x == 0:
            raise ValueError
        return x

    def __iter__(self):
        return self

    def __next__(self):
        self._i += 1
        if self._i == len(self.data):  # out of instances
            self._i = -1               # reset the iterable
            raise StopIteration        # stop the iteration
        try:
            return self.getData(self._i)
        except ValueError:
            return next(self)

You would use it like:
>>> trainLoader = DataLoader(DS(), batch_size=1, shuffle=False)
>>> for x in trainLoader:
...    print(x)
tensor([1])
tensor([2])
tensor([2])
...
tensor([1])
tensor([1])

Here all 0 instances have been skipped in the iterable dataset.
You can adapt this simple example to fit your needs.


Instruction: Intution behind weighted random sampler in PyTorch

Input:
I am trying to use WeightedRandomSampler for handling imbalance in the dataset (class1: 2555, class 2: 227, class 3: 621, class 4: 2552 images). However, I debugged the steps but the intuition behind it is not clear to me. My target labels are in form of one-hot encoded vectors as below.
train_labels.head(5)

I converted the labels to class index as:
labels = np.argmax(train_labels.loc[:, 'none':'both'].values, axis=1)
train_labels = torch.from_numpy(labels)
train_labels
tensor([0, 0, 1,  ..., 1, 0, 0])

Below are the steps, I used to calculate for the weighted random sampler. Please correct me if I am wrong with the interpretation of any steps.

Count the number of samples per class in the dataset
class_sample_count = np.array(train_labels.value_counts())
class_sample_count
array([2555, 2552,  621,  227])

Calculate the weight associated with each class
weight = 1. / class_sample_count
weight
array([0.00039139, 0.00039185, 0.00161031, 0.00440529])

Calculate the weight for each of the samples in the dataset.
samples_weight = np.array(weight[train_labels])
print(samples_weight[1], samples_weight[2] )
0.0003913894324853229 0.00039184952978056425

Convert the np.array to tensor
     tensor([0.0004, 0.0004, 0.0004,  ..., 0.0004, 0.0004, 0.0004],
     dtype=torch.float64)

After conversion to tensor, all the samples appear to have the same value in all four entries? Then how does Weighted Random Sampling is helping to deal with the imbalanced dataset?
I will be grateful for the help. Thank you.

Output:
This is because you are computing the weights on the one-hot encodings, and since there are four components (four classes) you end up with four identical weights per instance after the indexing weight[train_labels]. The fact that you have identical weights is perfectly fine because each instance should be assigned a unique weight. To the sampler, this weight corresponds to the probability of picking this instance. If a given class is prominent in the dataset, the associated frequency (i.e. the weight) will be low, and as such instances of that class will have a lower probability of getting sampled from the dataset.
With a fairly large number of samples, the goal with this weighting scheme is to have a balanced sampling even though class representations are imbalanced.
If you stick with one-hot encodings, you can just pick the first column:
>>> sample_weights = np.array(weight[train_labels])[:,0]

Then use WeightedRandomSampler to construct a sampler:
>>> sampler = WeightedRandomSampler(sample_weights, len(train_labels))

Finally you can plug it into a dataloader:
>>> DataLoader(dataset, batch_size, sampler=sampler)


Instruction: PyTorch TypeError: flatten() takes at most 1 argument (2 given)

Input:
I am trying to run this program in PyTorch which is custom:
class NeuralNetwork(nn.Module):
    def __init__(self):
        super(NeuralNetwork, self).__init__()
        self.flatten = nn.Flatten(start_dim=1)
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(2142, 51),
            nn.ReLU(),
            nn.Linear(51, 1)
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits

model = NeuralNetwork().to(device)

The above code is the custom NeuralNetwork. I defined the training loop below.
def train_loop(data, y, model, loss_fn, optimizer):
    for i in range(data.shape[0]):
        # Compute model prediction and loss
        pred = model(data[i, :, :])
        loss = loss_fn(pred, y[i, :])

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        print("Loss: {}".format(loss.item()))

The following is how I would like to train it
final_data = torch.randn(500, 42, 51)
final_output = torch.randn(500, 1)
learning_rate = 1e-3
batch_size = 1
epochs = 5
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

epochs = 10
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train_loop(final_data, final_output, model, loss_fn, optimizer)

print('Done!!')

The variable final_data is of shape 500x42x51. The variable final_output is of shape 500x1.
I have been trying to run the above data for 10 epochs but I always end up with this error:
Epoch 1
-------------------------------
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-17-3fb934698ecf> in <module>()
      9 for t in range(epochs):
     10     print(f"Epoch {t+1}\n-------------------------------")
---> 11     train_loop(final_data, final_output, model, loss_fn, optimizer)
     12
     13 print('Done!!')

4 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/flatten.py in forward(self, input)
     38
     39     def forward(self, input: Tensor) -> Tensor:
---> 40         return input.flatten(self.start_dim, self.end_dim)
     41
     42     def extra_repr(self) -> str:

TypeError: flatten() takes at most 1 argument (2 given)

The output is basically a classification between 0 or 1.
I am still a newbie in terms of PyTorch and would like some help solving this issue and understanding what's wrong.
Thank you

Output:
The error is a bit misleading, if you try running the code from a fresh kernel, the issues are elsewhere...
There are multiple issues with your code:

You are not using the correct shape for the target tensor y, here it should have a single dimension since the output tensor is two-dimensional.

The target tensor should be of dtype Long

When iterating over you data and selecting input (and target) with data[i, :, :] (and y[i, :]), you are essentially removing the batch axis. However all builtin nn.Module work with a batch axis. You can do a slice to avoid that side effect: with data[i:i+1] and y[i:i+1] respectively. Also do note that x[j, :, :] is identical to x[j].

That being said, the usage of the cross-entropy loss is not justified. You are outputting a single logit, so it doesn't make sense to use a cross-entropy loss.

You can either output two logits on the last layer of your model,

or switch to another loss function such as a binary cross-entropy loss (either using nn.BCELoss or a nn.BCEWithLogitsLoss which includes a sigmoid activation).
In this case the target vector should be of dtype float, and its shape should equal that of pred.

def train_loop(data, y, model, loss_fn, optimizer):
    for i in range(data.shape[0]):
        pred = model(data[i:i+1])
        loss = loss_fn(pred, y[i:i+1].float())
        # [...]

final_data = torch.randn(500, 42, 51)
final_output = torch.randint(0, 2, (500,1))
loss_fn = nn.BCEWithLogitsLoss()

Then the following will work:
>>> train_loop(final_data, final_output, model, loss_fn, optimizer)


Instruction: How can I build a neural network with 2 independent sets of weights and 2 losses?

Input:
Consider the following neural network:
import torch
import torch.nn as nn
import torch.optim as optim

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.blue_1 = nn.Linear(2, 10)
        self.red_1 = nn.Linear(10, 5)
        self.blue_2 = nn.Linear(5, 4)
        self.red_2 = nn.Linear(4, 3)
        self.blue_3 = nn.Linear(3, 2)
        self.red_3 = nn.Linear(2, 1)

    def forward(self, x):
        x = torch.relu(self.blue_1(x))
        x = self.red_1(x)
        x = self.blue_2(x)
        x = self.red_2(x)
        x = self.blue_3(x)
        x = self.red_3(x)
        return x

net = Model()

opt = optim.Adam(net.parameters())

features = torch.rand((10,2)) #10 inputs, each of 2D

for epoch in range(3):
    x = net(features)
    loss = torch.sum(torch.randint(0,10,(10,)) - x)
    loss.backward()
    print(loss)

I have 2 independent sets of weights, call them blues (e.g self.blue_1) and reds (e.g self.red_1). These 2 sets need to be multiplied in some combination (e.g see forward method). However, in comparison to what I have above I need the blue weights to be updated according to a certain loss function (e.g loss_blue = some_loss_function, and the red weights to be updated according to another loss function (e.g torch.sum(torch.randint(0,10,(10,)) - x)).
It's important that the red loss doesn't propagate through the blue weights and vice versa.
Is there a way to do this? I was thinking to even break it into 2 neural networks but I'm not sure if 1. it's the best approach, and 2. how to do it using that approach.

Output:
One way to do this would be, in four steps:

Do the first inference to compute loss_blue.
Backpropagate from loss_blue and update the blue parameters,
Then infer again to compute loss_red with the updated blue weights
Backpropagate from loss_red and update the red parameters.

This is similar to how you would go about training a GAN alternating between generator and discriminator with successive backward passes and updates. Having two optimizers handling the two sets of parameters makes it easier to use. Don't forget to clear the gradients before backpropagating so one loss doesn't pollute the other parameters with its gradient.

Something like this should work out:
net = Model()

optim_blue = optim.Adam(net.blues()) # fetch all blue parameters
optim_red = optim.Adam(net.reds())   # fetch all red parameters

features = torch.rand((10,2)) #10 inputs, each of 2D

# # inference, backprop and update on blue params
out = net(features)
loss_blue = torch.sum(torch.randint(0,10,(10,)) - out)

optim_red.zero_grad()
optim_blue.zero_grad()
loss_blue.backward()
optim_blue.step()

# # inference backprop and update on red params
out = net(features)
loss_red = out.mean()

optim_red.zero_grad()
optim_blue.zero_grad()
loss_red.backward()
optim_red.step()

Edit based on comment:

How do you specify which parameters/layers will be in the optimizer(optim_blue = optim.Adam(net.blues())). Is it something along the lines of optim_blue = optim.Adam([net. blue_1, net. blue_2...])?

Yes, something like that. For defining the optimizers, you can for example create two functions inside your model: reds and blues.
    def reds(self):
        return [*self.red_1.parameters(),
                *self.red_2.parameters(),
                *self.red_3.parameters()]

    def blues(self):
        return [*self.blue_1.parameters(),
                *self.blue_2.parameters(),
                *self.blue_3.parameters()]

What makes the gradients separated when you call loss_blue.backward()? That is, what stops them to flow through the reds? Is that the purpose of the 2 optimizers?

When loss_blue.backward, the gradient flows through all parameters of the model, including the red parameters. What makes all the difference is indeed the fact that optim_blue will only update the blue parameters, not the red ones.


Instruction: Fine Tuning Pretrained Model MobileNet_V3_Large PyTorch

Input:
I am trying to add a layer to fine-tune the MobileNet_V3_Large pre-trained model. I looked around at the PyTorch docs but they don't have a tutorials for this specific pre-trained model. I did find that I can fine-tune MobileNet_V2 with:
model_ft =models.mobilenet_v2(pretrained=True,progress=True)

model_ft.classifier[1] = nn.Linear(model_ft.last_channel, out_features=len(class_names))

but I am not sure what the linear layer for MobileNet V3 should look like.

Output:
For V3 Large, you should do
model_ft = models.mobilenet_v3_large(pretrained=True, progress=True)
model_ft.classifier[-1] = nn.Linear(1280, your_number_of_classes)

(This would also work for V2, but the code you posted would not work for V3 correctly).
To see the structure of your network, you can just do
print(model_ft.classifier)

or
print(model_ft)

For fine-tuning people often (but not always) freeze all layers except the last one. Again, the layer to not freeze is model_ft.classifier[-1] rather than model_ft.classifier[1].
Whether or not you should freeze layers depends on how much data you have, and is best determined empirically.


Instruction: RuntimeError: Trying to backward through the graph a second time

Input:
I'm trying to train 'trainable Bernoulli distribution' using 'pyro'.
I want to train Bernoulli distribution's parameter(probability to win) using NLL loss.
train_data is one-hot encoded sparse matrix(2034,19475) and train_labels has 4 value(4 class, [0,1,2,3]).

import torch
import pyro
pyd = pyro.distributions

print("torch version:", torch.__version__)
print("pyro version:", pyro.__version__)

import matplotlib.pyplot as plt
import numpy as np
torch.manual_seed(123)

### 0. define Negative Log Likelihood(NLL) loss function
def nll(x_train, distribution):
    return -torch.mean(distribution.log_prob(torch.tensor(x_train, dtype=torch.float)))

### 1. initialize bernoulli distribution(trainable distribution)
train_vars = (pyd.Uniform(low=torch.FloatTensor([0.01]),
                          high=torch.FloatTensor([0.1])).rsample([train_data.shape[-1]]).squeeze())
distribution = pyd.Bernoulli(probs=train_vars)

### 2. initialize 'label 0' data
class_mask = (train_labels==0)
class_data = train_data[class_mask, :]

### 3. initialize optimizer
optim = torch.optim.Adam([train_vars])

train_vars.requires_grad=True

### 4. train loop
for i in range(0,100):

    loss = nll(class_data, distribution)

    loss.backward()

When I run this code, I get RUNTIME ERROR like below..
How should I deal with this error case?
your comment would be very very very appreciate.
torch version: 1.9.0+cu102
pyro version: 1.7.0
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-269-0081bb1bb843> in <module>
     25     loss = nll(class_data, distribution)
     26
---> 27     loss.backward()
     28

/nf/yes/lib/python3.8/site-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    253                 create_graph=create_graph,
    254                 inputs=inputs)
--> 255         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    256
    257     def register_hook(self, hook):

/nf/yes/lib/python3.8/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    145         retain_graph = create_graph
    146
--> 147     Variable._execution_engine.run_backward(
    148         tensors, grad_tensors_, retain_graph, create_graph, inputs,
    149         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag

RuntimeError: Trying to backward through the graph a second time (or directly access saved variables after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved variables after calling backward.

Output:
You need to move
distribution = pyd.Bernoulli(probs=train_vars)

inside the loop, because it uses train_vars, which requires_grad.


Instruction: Some strange questions about pytorch copy a tensor

Input:
I am a bit confused about pytorch's shared memory mechanism.
a = torch.tensor([[1,0,1,0],
                                   [0,1,1,0]])
b = a
b[b == 1] = 0

It's easy to know that a and b will simutaneously become tensor([[0,0,0,0],[0,0,0,0]]), cause a and b share the same memory. When I changed the code to
a = torch.tensor([[1,0,1,0],
                                   [0,1,1,0]])
b = a
b = b - 1

b became tensor([[0,-1,0,-1],[-1,0,0,-1]]), but a is still torch.tensor([[1,0,1,0],[0,1,1,0]]).
a and b are sharing the same memory. Why did b changed, while a didn't change?

Output:
In your second example a and b share the same reference but b = b - 1 is actually a copy. You are not affecting the underlying data of b (and not of a neither since it's the same).

You can look at it this way:
>>> a = torch.tensor([[1,0,1,0],
                      [0,1,1,0]])
>>> b1 = a
>>> b2 = b1 - 1

Comparing their pointer to the data buffers:
>>> a.data_ptr() == b1.data_ptr()
True

>>> b1.data_ptr() == b2.data_ptr()
False

If in fact, you operate on b inplace, you will of course change a as well:
>>> a = torch.tensor([[1,0,1,0],
                      [0,1,1,0]])
>>> b1 = a
>>> b1.sub_(1)

Then you haven't made a copy:
>>> a.data_ptr() == b1.data_ptr()
True


Instruction: Understanding of Pytorch NLLLOSS

Input:
PyTorch's negative log-likelihood loss, nn.NLLLoss is defined as:

So, if the loss is calculated with the standard weight of one in a single batch the formula for the loss is always:

-1 * (prediction of model for correct class)

Example:

Correct Class = 0
prediction of model for correct class = 0.5
loss = -1 * 0.5
So, why is it called the "negative log-likelihood loss", if there isn't a log function involved in calculating the loss?

Output:
Indeed no log is being used to compute the result of nn.NLLLoss so this can be a little confusing. However, I believe the reason why it was called this way is because it expects to receive log-probabilities:

The input given through a forward call is expected to contain log-probabilities of each class. - docs

In the end it does not make much sense to have it in the name since you might as well want to apply this function on non-log-probabilities...


Instruction: When I make pytorch attention Can you give me a idea to fix loss

Input:
Here is my model
class Build_Model(nn.Module):
    def __init__(self,args) :
    super(Build_Model, self).__init__()
    self.hidden_size = args.dec_size
    self.embedding = nn.Embedding(args.n_vocab, args.d_model)
    self.enc_lstm = nn.LSTM(input_size =args.d_model, hidden_size=args.d_model,batch_first=True)
    self.dec_lstm = nn.LSTM(input_size =args.d_model, hidden_size=args.d_model,batch_first=True)
    self.soft_prob = nn.Softmax(dim=-1)
    self.softmax_linear = nn.Linear(args.d_model*2,len(vocab))
    self.softmax_linear_function = nn.Softmax(dim = -1)

def forward(self, enc_inputs, dec_inputs) :
    enc_hidden = self.embedding(enc_inputs)
    dec_hidden = self.embedding(dec_inputs)
    enc_hidden , (enc_h_state,enc_c_state) = self.enc_lstm(enc_hidden)
    dec_hidden,(dec_h_state,dec_c_state) = self.dec_lstm(dec_hidden,(enc_h_state,enc_c_state))
    attn_score = torch.matmul(dec_hidden, torch.transpose(enc_hidden,2,1))
    attn_prob  = self.soft_prob(attn_score)
    attn_out = torch.matmul(attn_prob,enc_hidden)
    cat_hidden = torch.cat((attn_out, dec_hidden),-1)
    y_pred = self.softmax_linear_function(self.softmax_linear(cat_hidden))
    y_pred = torch.argmax(y_pred,dim =-1)
    print('y_pred = ',y_pred.shape)
    y_pred = y_pred.view(-1, 150)
    print('2y_pred = ',y_pred.shape)
    return y_pred

Here is the loss function
def lm_loss(y_true, y_pred):
    print(y_pred.shape)
    y_pred_argmax = y_pred
    #y_pred_argmax = y_pred_argmax.view(-1,150)
    print(y_true.shape, y_pred_argmax.shape)
    criterion = nn.CrossEntropyLoss(reduction="none")
    loss = criterion(y_true.float(), y_pred_argmax.float()[0])
    #mask = tf.not_equal(y_true, 0)
    mask = torch.not_equal(y_pred_argmax,0)
    #mask = tf.cast(mask, tf.float32)
    mask = mask.type(torch.FloatTensor).to(device)
    loss *= mask
    #loss = tf.reduce_sum(loss) / tf.maximum(tf.reduce_sum(mask), 1)
    loss = torch.sum(loss) / torch.maximum(torch.sum(mask),1)
    return loss

The last is evaluation
optimizer.zero_grad()
print(train_enc_inputs.shape,train_dec_inputs.shape, train_dec_labels.shape )
y_pred = model(train_enc_inputs,train_dec_inputs)
#y_pred = torch.argmax(y_pred,dim =-1)
print(y_pred.shape )
loss = lm_loss(train_dec_labels, y_pred)

The output is here:
torch.Size([32, 120]) torch.Size([32, 150]) torch.Size([32, 150])
y_pred =  torch.Size([32, 150])
2y_pred =  torch.Size([32, 150])
torch.Size([32, 150])
torch.Size([32, 150])
torch.Size([32, 150]) torch.Size([32, 150])

The error traceback:
ValueError Traceback (most recent call last)
<ipython-input-159-cc8976139dd5> in <module>()

      9     #y_pred = torch.argmax(y_pred,dim =-1)
     10     print(y_pred.shape )
---> 11     loss = lm_loss(train_dec_labels, y_pred)
     12     n_step += 1
     13     if n_step % 10 == 0:

3 frames

<ipython-input-158-39ba03042d04> in lm_loss(y_true, y_pred)

     15     print(y_true.shape, y_pred_argmax.shape)
     16     criterion = nn.CrossEntropyLoss(reduction="none")
---> 17     loss = criterion(y_true.float(), y_pred_argmax.float()[0])
     18     #mask = tf.not_equal(y_true, 0)
     19     mask = torch.not_equal(y_pred_argmax,0)

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)

   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)

   1119     def forward(self, input: Tensor, target: Tensor) -> Tensor:
   1120         return F.cross_entropy(input, target, weight=self.weight,
-> 1121                                ignore_index=self.ignore_index, reduction=self.reduction)
   1122
   1123

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)

   2822     if size_average is not None or reduce is not None:
   2823         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2824     return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2825
   2826

ValueError: Expected input batch_size (32) to match target batch_size (150).

How can I fix it?

Output:
There are a few issues with your usage of nn.CrossEntropyLoss:

You are supposed to call nn.CrossEntropyLoss with criterion(y_pred, y_true), you seem to have switched the two.

y_pred contains the output logits of your network i.e. it hasn't been passed through a softmax: you need to remove self.softmax_linear_function in your model)

Also y_pred should contain all components and not be the results of an argmax.

y_true is passed in dense format: it contains the true class labels and has one dimension less than the prediction y_pred.


Instruction: How to estimate the run time of a neural network on mobile phones?

Input: I trained a neural network with approximately 26000 parameters, Im intending to use it on mobile phones for real time inference. Im wondering if there is a way to estimate the run time of a neural network given the size of the network and the operating device.

Output:
You need to estimate the number of floating point ops (FLOPS) that running your model requires. For example, multiplying two N x N matrices counts as 2 N^3 FLOPS. (There are software packages that help you do that in PyTorch) 1 multiply-add counts as 2 FLOPS, by the way.
Then, you need to know the capabilities of you target device. How many floating point ops can it do per second? This provides an upper bound on how fast your code can run. Will your code reach this theoretical limit? That is unclear, but this gives you something to shoot for. The simpler the problem (small tensors), the more likely you are to fall behind this limit.
If you quantize your models for deployment, you need to adjust your calculation accordingly (use the speed of the relevant integer operations).


Instruction: When training a Deep learning model on multiple datasets, is it better concatenating all datasets and shuffling it, or train datasets sequentially?

Input: So lets say that I Have datasets A, B, and C to train a model. My current solution take batches randomly from A, then from B, then from C. I wonder if concatenating all datasets and shuffling so that training would be more random would improve results

Output: As you pointed out in your comment, the samples in the datasets are drawn from slightly different "distributions" (e.g., real vs synthetic images). In this case, it is better to randomly sample points from all datasets for each batch, rather than going sequentially through the different datasets.


Instruction: Import "torch" could not be resolved

Input:
Using vscode 1.60.2
Running this code in command line, after executing the "python" command works.  So I know that the library is properly installed.  This is a problem specifically with vscode.  I restarted vscode, and even restarted windows after installing pytorch with pip.  Did not fix it.
import torch

# Model
model = torch.hub.load('ultralytics/yolov5', 'yolov5s', pretrained=True)

# Images
imgs = ['https://ultralytics.com/images/zidane.jpg']  # batch of images

# Inference
results = model(imgs)

# Results
results.print()
results.save()  # or .show()

results.xyxy[0]  # img1 predictions (tensor)
results.pandas().xyxy[0]  # img1 predictions (pandas)
#      xmin    ymin    xmax   ymax  confidence  class    name
# 0  749.50   43.50  1148.0  704.5    0.874023      0  person
# 1  433.50  433.50   517.5  714.5    0.687988     27     tie
# 2  114.75  195.75  1095.0  708.0    0.624512      0  person
# 3  986.00  304.00  1028.0  420.0    0.286865     27     tie

Output: You might have changed which interpreter you're using for Python.  In VSCode go to the command palette and search for Python: Select Interpreter.  If there's a default option go with that since that's where you might have installed the module to using pip.  Should look something like this.  Command palette can be reached by Ctrl+Shift+p.  If that doesn't work set up a virtual environment for your project and install your module there.


Instruction: PyCox: Feature transforms with DataFrameMapper

Input:
I am trying to implement DeepSurv for survival analysis with the Python package pycox.
The author of the package also provide also a notbook with a coding example so I tried to transfer the code to my data.
However, there seems to be a problem defining x_train due to their proposed Feature transforms with DataFrameMapper.
In the notbook it says:
cols_standardize = ['x0', 'x1', 'x2', 'x3', 'x8']
cols_leave = ['x4', 'x5', 'x6', 'x7']

standardize = [([col], StandardScaler()) for col in cols_standardize]
leave = [(col, None) for col in cols_leave]

x_mapper = DataFrameMapper(standardize + leave)

x_train = x_mapper.fit_transform(df_train).astype('float32')
x_val = x_mapper.transform(df_val).astype('float32')
x_test = x_mapper.transform(df_test).astype('float32')

In the notbook they are standardizing the 5 numerical covariates but I have nothing to standardize. So I changed the code into:
cols_standardize = []
cols_leave = df_train.columns.values.tolist()

standardize = [([col], StandardScaler()) for col in cols_standardize]
leave = [(col, None) for col in cols_leave]

x_mapper = DataFrameMapper(standardize + leave)

x_train = x_mapper.fit_transform(df_train).astype('float32')
x_val = x_mapper.transform(df_val).astype('float32')
x_test = x_mapper.transform(df_test).astype('float32')

But when I execute training the model this error occurs:
batch_size = 256
lrfinder = model.lr_finder(x_train, y_train, batch_size, tolerance=10)
_ = lrfinder.plot()

RuntimeError: expected device cpu and dtype Float but got device cpu and dtype Long

Is it maybe because of the batch_size? What does batch_size actually mean?
However, I also tried to skip the whole Feature transforms step, so I just changed my dataframes into floats:
x_train = df_train.astype('float32')
x_val = df_val.astype('float32')
x_test = df_test.astype('float32')

But then if I go on training the modell it says:
All objects in 'data' doest have the same type.

I am really confused how to prepare my data to use pycox. Especially this label transforms step with standardization appears really confusing. I would be glad for any help!

Output: There is no problem with the code. Try to upgrade pytorch, like suggested here https://github.com/huggingface/transformers/issues/2126


Instruction: understanding transforms: resize and centercrop with same size

Input:
I am trying to understand this particular set of compose  transforms:
transform= transforms.Compose([transforms.Resize((224,224) interpolation=torchvision.transforms.InterpolationMode.BICUBIC),\
transforms.CenterCrop(224),transforms.ToTensor(),\
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])

Does it make sense (and is it legit) to do centercrop after transforms - with the same size parameter? I would have thought resize itself is giving the centercrop but I see in the repos that centercrop is composed after resize - both with exactly the same sizes. I wonder what is the use of doing such a thing. For the sake of completeness, I would like to add that my input image sizes vary (ie they are all not of the same dims).
thanks!

Output:
I would have thought resize itself is giving the center crop.

Function T.Resize won't center crop your image, the center will stay the same since you are only resizing the original image, i.e. proportions are kept and the original center remains at the center. Applying a crop of the same shape as the image - since it's just after the resize - with T.CenterCrop doesn't make any difference since you are cropping nothing out of the image.
If you change the sizes of your T.CenterCrop, then this and the order you apply both transforms will matter greatly.


Instruction: install pytorch c++ api CUDA11.4 for ubuntu

Input:
I'm trying to use the PyTorch c++ API on an ubuntu 18.04. I've installed CUDA 11.4 and cuDNN 8.2.4.15. The source I'm compiling is available here. compiling CUDA with nvcc works and the cuDNN installation test succeeds. But I am unable to find a good documentation for installing and compiling projects with PyTorch c++ api on Ubuntu. Do you know any god ones?
system configurations:
OS: ubuntu 18.04
GPU: 1 x NVIDIA Tesla P4
Machine Type: n1-standard-2 (2 vCPUs, 7.5 GB memory) on google cloud console
CUDA: 11.4
cuDNN: 8.2.4.15

Output:
I used the next ones to install on Ubuntu 16, it can be helpful for you.

PyTorch C++ API Ubuntu Installation Guide
tutorial to compile and use pytorch on ubuntu 16.04

Also can it will be util to refer the official documentation to use PyTorch c++ for Linux systems and the GCPdocumentation.


Instruction: Perceptron with weights of bounded condition number

Input:
Let N be a (linear) single-layer perceptron with weight matrix w of dimension nxn.
I want to train N under the Boolean constraint that the condition number k(w) of the weights w remain below a given threshold k_0 at each step of the optimisation.
Is there a standard way to implement this constraint (in pytorch, say)?

Output:
After each optimizer step, go through the list of parameters and recondition all matrices:
(code looked at for a few seconds, but not tested)
def recondition_(x, max_cond): # would need to be fixed for non-square x
    u, s, vh = torch.linalg.svd(x)
    curr_cond = s[0] / s[-1]
    if curr_cond > max_cond:
        ratio = curr_cond / max_cond
        mult = torch.linspace(0, math.log(ratio), len(s)).exp()
        s = mult * s
        x[:] = torch.mm(u, torch.mm(torch.diag(s), vh))

Training loop:
...
optimizer.step()

with torch.no_grad():
    for p in model.parameters():
        if p.dim() == 2:
            recondition_(p, max_cond)
...


Instruction: pytorch: NLLLoss ignore_index default value

Input:
in the pytorch NLLLoss doc the default of ignore_index is -100 instead of the usual None, are there any particular reasons? seems like any negative value is equivalent.
BTW, what may be the reason that I would want to ignore an index? Thanks!

Output:
The value for ignore_index must be an int, that's why the default value is an int and not None. The default value is arbitrary, it could have been any negative number, i.e. anything that is not a "valid" class label. The function will ignore all elements for which the target instance has that class label. In practice, this option can be used to identify unlabeled pixels for example in dense prediction tasks.

Edit: Tracing back the implementation of nn.NLLLoss, we can find this comment in the nll_loss implementation of torch/onnx/symbolic_opset12.py:
# in onnx NegativeLogLikelihoodLoss specification, ignore_index is optional without default value.
# therefore we need to set ignore_index attribute even if it is not specified (e.g. ignore_index=-100).
ignore_index = sym_help._maybe_get_const(ignore_index, "i")


Instruction: How can I control (or at least record) parameters passed to torchvision transform on an image by image basis?

Input: I am studying the effects of blur and noise on an image classifier, and I would like to use torchvision transforms to apply varied amounts of Gaussian blur and Poisson noise my images. It's pretty trivial to specify a probability distribution for the noise and blur parameters, but I can't figure out how to either control those parameters on an image by image basis or get PyTorch to record the parameters actually used for each image. Could I do this by defining my transform inside the dataset class rather than passing it to the dataloader, so that each time I load an image a custom transform is created and it's parameters are returned with the image and its label?

Output:
The transform mechanism provided by PyTorch uses simple callable objects that are called automatically upon loading samples from Dataset. There is nothing fundamentally stopping you from doing all your transforms from Dataset itself. Since you haven't provided any code, I can only offer some pseudocodes
from torchvision.transforms.functional import gaussian_blur

class CoolDataset(Dataset):
    def __init__(self, root_dir):
        self.image_list = os.listdir(root_dir)
        self.sample_wise_blur_std = [0.1, 0.2, ..]

    def __getitem__(self, i):
        img = read_image(self.image_list[i]) # .shape is (C,H,W)
        blurred = gaussian_blur(img, (3,3), std=self.sample_wise_blur_std[i])
        return img, self.sample_wise_blur_std[i]

with transform parameters fused in your Dataset definition, your Dataloader will collate them for you
cooldl = DataLoader(CoolDataset('/path/to/images'), ...)
for X, blurs in cooldl:
    # X.shape is (B,C,H,W)
    # blurs.shape is (B,)
    pass

I hope this is what you are looking for.


Instruction: torch.hub.load() raises HTTPError: HTTP Error 404: Not Found when loading model

Input:
I had this simple piece of code found on the fairseq GitHub repository which basically loads the bart.base PyTorch model from torch.hub:
bart = torch.hub.load('pytorch/fairseq', 'bart.base')

This code was working perfectly around two weeks ago, now it raises the following error despite the fact that I didn't change anything:
HTTPError                                 Traceback (most recent call last)
<ipython-input-7-68181b5f094c> in <module>()
      1 # torch.cuda.empty_cache()
----> 2 bart = torch.hub.load('pytorch/fairseq', 'bart.base') #takes around two minutes
      3 # bart.cuda()  # use GPU
...
...
/usr/lib/python3.7/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
--> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 404: Not Found

Also, I found out that this happens with other models on fairseq. All the following models raise the same error:
>>> torch.hub.load('pytorch/fairseq', 'transformer.wmt16.en-de')
# ERROR!

>>> torch.hub.load('pytorch/fairseq', 'camembert')
# ERROR!

So, there must be something in common among all of them.

Output:
Apparently, the fairseq folks decided to change the default branch of their GitHub repository from master to main exactly 7 days ago. (check this commit).
So, adding the main branch to the repo info will fix the problem:
bart = torch.hub.load('pytorch/fairseq:main', 'bart.base') #<--- added :main

And that's because in torch.hub.load() function the default branch name is master. So actually, you were calling pytorch/fairseq:master which doesn't exist anymore.
And all other models are working now:
torch.hub.load('pytorch/fairseq:main', 'transformer.wmt16.en-de')
# WORKS!

>>> torch.hub.load('pytorch/fairseq:main', 'camembert')
# WORKS!


Instruction: Generator not working as expected in python

Input:
I've been working on a genetic algorithm in PyTorch, and I've run into an issue while trying to mutate my model's parameters. I've been using the .apply() function to randomly change a model's weights and biases. Here is the exact function I made:
def mutate(m):
    if type(m) == nn.Linear:
        m.weight = nn.Parameter(m.weight+torch.randn(m.weight.shape))
        m.bias = nn.Parameter(m.bias+torch.randn(m.bias.shape))

This function does work for sure, I've tested it, but this isn't the weird part. While trying to use this function for every model in a list, the same mutation happens to each and every model. I obviously don't want this, as I want variety in my population. Here is a reproduceable example:
import torch
import torch.nn as nn
population_size = 5 #Size of the population
population = [nn.Linear(1,1)]*population_size #Creating my population, each agent is a player in this list
dummy_input = torch.rand(1) #Random input
def mutate(m): #Mutation function
    if type(m) == nn.Linear:
        m.weight = nn.Parameter(m.weight+torch.randn(m.weight.shape))
        m.bias = nn.Parameter(m.bias+torch.randn(m.bias.shape))
population = list(x.apply(mutate) for x in population) #This is the line I've been having issues with
for i in population:
    print (i(dummy_input)) #This is here to show that all the models are mutating in the same way and outputting the same thing

This code has the following output:
tensor([-2.0366], grad_fn=<AddBackward0>)
tensor([-2.0366], grad_fn=<AddBackward0>)
tensor([-2.0366], grad_fn=<AddBackward0>)
tensor([-2.0366], grad_fn=<AddBackward0>)
tensor([-2.0366], grad_fn=<AddBackward0>)

As you can see, all the models mutated in the same way, and are yielding the same output.
This is running in Python 3.9, thank you all in advance.

Output:
It looks like creating a list of linear parameters in the way you do simply copies the initial nn.Linear object. For example, setting population[0].weight = nn.Parameter() sets all linear layer weights in population list to an empty parameter value. In your case, the final random weight and bias assigned by the mutate function is given to all layers in the population list since they are all copies of one another.
Changing the fourth line of your code to population = [nn.Linear(1,1) for _ in range(population_size)] creates five unique linear layers and fixes this problem.


Instruction: Keras - bilinear data transformation (like in pytorch)

Input:
So I'm trying to implement in Keras (tf) "bilinear transformation to the incoming data" (coming from pytorch). Transformation is defined in torch.nn.Bilinear as
y = x_1^T A x_2 + b
I built custom layer in Keras that in method call() will return transformed data (I'd like to use this layer as a part of my model later on). I do have however problem with implementation of the transformation function itself due to 3d input I have.
and shapes are as follows:

x_1= TensorShape([72, 10, 6])
x_2= TensorShape([72, 10, 6])
self.w = TensorShape([24, 6, 6]) (24 coming from defined out_features=4*feature_vector)
self.b = TensorShape([24])

output I'd like to use as an input for LSTM layer.
So my formula in tf looks like that:
a = tf.matmul(tf.matmul(input1, self.w), input2) + self.b
and I tried transposing different parts but it doesn't work - I have a problem with not compatible dimensions and I'd really appreciate any hint

Output:
I have a detailed implementation of bilinear with pytorch and you can translate to keras easily
a = torch.randn(2,3) # input1
b = torch.randn(3,7,5) # bilinear matrix weight (input1 dim, input2 dim, output dim)
c = torch.randn(2,7) # input2

B = nn.Parameter(b)
t = a @ B.permute(2,0,1)
output = (t * c).sum(-1).t()


Instruction: What is the last line of this Rnn function means?

Input:
I am here to ask a noob question.
class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_classes):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_size*2, num_classes)

    def forward(self, x):
        h0 = torch.zeros(self.num_layers*2, x.size(0), self.hidden_size).to(device)
        out, _ = self.rnn(x, h0)  # out: tensor of shape (batch_size, seq_length, hidden_size)
        out = self.fc(out[:, -1, :])
        return out

Here what does out = self.fc(out[:, -1, :]) means? And also why there is a "_" in out, _ = self.rnn(x, h0) ?

Output:
The line out = self.fc(out[:, -1, :]) is using negative indexing: out is a tensor of shape batch_size x seq_length x hidden_size, so out[:, 1, :] would return the first element along the second dimension (or axis), and out[:, -1, :] returns the last element along the second dimension. It would be equivalent to out[:, seq_length-1, :].
The underscore in out, _ = self.rnn(x, h0) means that self.rnn(x, h0) returns two outputs, and out is assigned to the first output, and the second output isn't assigned to anything so _ is a placeholder.


Instruction: Pytorch: TypeError: list is not a Module subclass

Input:
I want to extract some layers from a model, so I write
nn.Sequential(list(model.children())[:7])

but get error: list is not a Moudule subclass. I know I should write
nn.Sequential(*list(model.children)[:7])

but why I must add the * ???
If I just wanna to get the [] including the layers, I must write
layer = list(model.children())[:7]
but not
layer = *list(model.children())[:7]

In this situation, the * does not work and get error
    layer = *list(model_ft.children())[:3]
           ^
SyntaxError: can't use starred expression here

Why???

Output:
list(model.children())[:7] returns a list, but the input of nn.Sequential() requires the modules to be an OrderedDict or to be added directly, not in a python list.

nn.Sequential Modules will be added to it in the order they are passed in the constructor. Alternatively, an OrderedDict of modules can be passed in.

# nn.Sequential(list(model.children)[:3]) means, which is wrong
nn.Sequential([Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)),
ReLU(inplace=True),
MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)])

# nn.Sequential(*list(model.children)[:3]) mean
nn.Sequential(Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3)),
ReLU(inplace=True),
MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False))

That's why you need to unpack the list using *. It can only be used inside a function, hence, it doesn't work in your last case. Read * in python


Instruction: Concat two tensors

Input:
I want to concat two tensors of size a: torch.Size([16, 1]) and b: torch.Size([16, 120])
to be of size torch.Size([16, 121])
could you please help with that?

Output:
Here you can use the torch.cat() function.
Example:
>>> a = torch.rand([16,1])
>>> b = torch.rand([16,120])
>>> a.size()
torch.Size([16, 1])
>>> b.size()
torch.Size([16, 120])
>>> c = torch.cat((a,b),dim=1)
>>> c.size()
torch.Size([16, 121])

What you want to do is to concatenate the tensors on the first dimension (dim=1).


Instruction: AttributeError: 'ReLU' object has no attribute 'dim'

Input:
I am building a GAN, and my discriminator function is defined as
class Discriminator(nn.Module):
def __init__(self):
    super(Discriminator, self).__init__()

    self.fc1 = nn.Linear(50*15, 32)
    self.fc2 = nn.Linear(32, 32)
    self.fc3 = nn.Linear(32, 1)

def forward(self, x):
    x = x.flatten()
    x = torch.nn.ReLU(self.fc1(x))
    x = torch.nn.ReLU(self.fc2(x))

    return torch.nn.Sigmoid(self.fc3(x))

When I was testing the code, I got an error with the following command
discriminator(gen_series)

where gen_series is a tensor with the dimension 15*50. The error occurs as
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-99-fa68eff35865> in <module>
     16 valid = Variable(Tensor(piece, time).fill_(1.0), requires_grad=False)
     17 print(gen_series)
---> 18 discriminator(gen_series)
     19 # g_loss = adversarial_loss(discriminator(gen_series), valid)

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

<ipython-input-94-7c6c59da67f9> in forward(self, x)
     27         x = x.flatten()
     28         x = torch.nn.ReLU(self.fc1(x))
---> 29         x = torch.nn.ReLU(self.fc2(x))
     30
     31         return torch.nn.Sigmoid(self.fc3(x))

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/linear.py in forward(self, input)
     85
     86     def forward(self, input):
---> 87         return F.linear(input, self.weight, self.bias)
     88
     89     def extra_repr(self):

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
   1366         - Output: :math:`(N, *, out\_features)`
   1367     """
-> 1368     if input.dim() == 2 and bias is not None:
   1369         # fused op is marginally faster
   1370         ret = torch.addmm(bias, input, weight.t())

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in __getattr__(self, name)
    583                 return modules[name]
    584         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 585             type(self).__name__, name))
    586
    587     def __setattr__(self, name, value):

AttributeError: 'ReLU' object has no attribute 'dim'

I didn't find any related questions. Any help is appreciated!!

Output:
You can use this code and I think it would work well
class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        self.fc = nn.Sequential(
                  nn.Linear(50 * 15, 32),
                  nn.ReLU(),
                  nn.Linear(32, 32),
                  nn.ReLU(),
                  nn.Linear(32, 1),
                  nn.Sigmoid()
        )

    def forward(self, x):
        x = x.flatten()
        x = self.fc(x)

        return x