Instruction: IndexError: Target is out of bounds

Input:
I am currently trying to replicate the article
https://towardsdatascience.com/text-classification-with-bert-in-pytorch-887965e5820f
to get an introduction to PyTorch and BERT.
I used some own sample corpus and corresponding tragets as practise, but the code throws the following:
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-4-8577755f37de> in <module>()
    201 LR = 1e-6
    202
--> 203 trainer(model, df_train, df_val, LR, EPOCHS)

3 frames
<ipython-input-4-8577755f37de> in trainer(model, train_data, val_data, learning_rate, epochs)
    162                 output = model(input_id, mask)
    163
--> 164                 batch_loss = criterion(output, torch.max(train_label,1)[1])
    165                 total_loss_train += batch_loss.item()
    166

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1100         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1101                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102             return forward_call(*input, **kwargs)
   1103         # Do not call functions when jit is used
   1104         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)
   1150         return F.cross_entropy(input, target, weight=self.weight,
   1151                                ignore_index=self.ignore_index, reduction=self.reduction,
-> 1152                                label_smoothing=self.label_smoothing)
   1153
   1154

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
   2844     if size_average is not None or reduce is not None:
   2845         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2846     return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
   2847
   2848

IndexError: Target 32 is out of bounds.

The code is mostly identical to the one in the article, except of course the more extensive lable-dict.
Orginial:
labels = {'business':0,
          'entertainment':1,
          'sport':2,
          'tech':3,
          'politics':4
          }

Mine:
labels =
{'Macroeconomics': 0,
 'Microeconomics': 1,
 'Labor Economics': 2,
 'Subnational Fiscal Issues': 3,
 'Econometrics': 4,
 'International Economics': 5,
 'Financial Economics': 6,
 'Health, Education, and Welfare': 7,
 'Public Economics': 8,
 'Development and Growth': 9,
 'Industrial Organization': 10,
 'Other': 11,
 'Environmental and Resource Economics': 12,
 'History': 13,
 'Regional and Urban Economics': 14,
 'Development Economics': 15,
 'Corporate Finance': 16,
 'Children': 17,
 'Labor Studies': 18,
 'Economic Fluctuations and Growth': 19,
 'Economics of Aging': 20,
 'Economics of Education': 21,
 'International Trade and Investment': 22,
 'Asset Pricing': 23,
 'Health Economics': 24,
 'Law and Economics': 25,
 'International Finance and Macroeconomics': 26,
 'Monetary Economics': 27,
 'Technical Working Papers': 28,
 'Political Economy': 29,
 'Development of the American Economy': 30,
 'Health Care': 31,
 'Productivity, Innovation, and Entrepreneurship': 32}

Code:
class Dataset(torch.utils.data.Dataset):

    def __init__(self, df):

        self.labels = torch.LongTensor([labels[label] for label in df["category"]])
        self.texts = [tokenizer(text,
                               padding='max_length', max_length = 512, truncation=True,
                                return_tensors="pt") for text in df['text']]

    def classes(self):
        return self.labels

    def __len__(self):
        return len(self.labels)

    def get_batch_labels(self, idx):
        # Fetch a batch of labels
        return np.array(self.labels[idx])

    def get_batch_texts(self, idx):
        # Fetch a batch of inputs
        return self.texts[idx]

    def __getitem__(self, idx):
        batch_texts = self.get_batch_texts(idx)
        batch_y = np.array(range(0,len(labels)))

        return batch_texts, batch_y

#Splitting the sample into trainingset, validationset and testset (80,10,10)
np.random.seed(112)
df_train, df_val, df_test = np.split(df.sample(frac=1, random_state=42),
                                     [int(.8*len(df)), int(.9*len(df))])

print(len(df_train),len(df_val), len(df_test))

from torch import nn

class BertClassifier(nn.Module):

    def __init__(self, dropout=0.5):

        super(BertClassifier, self).__init__()

        self.bert = BertModel.from_pretrained('bert-base-cased')
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 5)
        self.relu = nn.ReLU()

    def forward(self, input_id, mask):

        _, pooled_output = self.bert(input_ids= input_id, attention_mask=mask,return_dict=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        final_layer = self.relu(linear_output)

        return final_layer

from torch.optim import Adam
from tqdm import tqdm

def trainer(model, train_data, val_data, learning_rate, epochs):

    train, val = Dataset(train_data), Dataset(val_data)

    train_dataloader = torch.utils.data.DataLoader(train, batch_size=2, shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)

    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")

    criterion = nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr= learning_rate)

    if use_cuda:

            model = model.cuda()
            criterion = criterion.cuda()

    for epoch_num in range(epochs):

            total_acc_train = 0
            total_loss_train = 0

            for train_input, train_label in tqdm(train_dataloader):

                train_label = train_label.to(device)
                mask = train_input['attention_mask'].to(device)
                input_id = train_input['input_ids'].squeeze(1).to(device)

                output = model(input_id, mask)

                batch_loss = criterion(output, torch.max(train_label,1)[1])
                total_loss_train += batch_loss.item()

                acc = (output.argmax(dim=1) == train_label).sum().item()
                total_acc_train += acc

                model.zero_grad()
                batch_loss.backward()
                optimizer.step()

            total_acc_val = 0
            total_loss_val = 0

            with torch.no_grad():

                for val_input, val_label in val_dataloader:

                    val_label = val_label.to(device)
                    mask = val_input['attention_mask'].to(device)
                    input_id = val_input['input_ids'].squeeze(1).to(device)

                    output = model(input_id, mask)

                    batch_loss = criterion(output, val_label)
                    total_loss_val += batch_loss.item()

                    acc = (output.argmax(dim=1) == val_label).sum().item()
                    total_acc_val += acc

            print(
                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / len(train_data): .3f} \
                | Train Accuracy: {total_acc_train / len(train_data): .3f} \
                | Val Loss: {total_loss_val / len(val_data): .3f} \
                | Val Accuracy: {total_acc_val / len(val_data): .3f}')

EPOCHS = 5
model = BertClassifier()
LR = 1e-6

trainer(model, df_train, df_val, LR, EPOCHS)

Output:
You're creating a list of length 33 in your __getitem__ call which is one more than the length of the labels list, hence the out of bounds error. In fact, you create the same list each time this method is called. You're supposed to fetch the associated y with the X found at idx.
If you replace batch_y = np.array(range(...)) with batch_y = np.array(self.labels[idx]), you'll fix your error. Indeed, this is already implemented in your get_batch_labels method.


Instruction: How to solve no such node error in pytables and h5py

Input:
I built an hdf5 dataset using pytables. It contains thousands of nodes, each node being an image stored without compression (of shape 512x512x3). When I run a deep learning training loop (with a Pytorch dataloader) on it it randomly crashes, saying that the node does not exist. However, it is never the same node that is missing and when I open the file myself to verify if the node is here it is ALWAYS here.
I am running everything sequentially, as I thought that I may have been the fault of multithreading/multiprocessing access on the file. But it did not fix the problem. I tried a LOT of things but it never works.
Does someone have an idea about what to do ? Should I add like a timer between calls to give the machine the time to reallocate the file ?
Initially I was working with pytables only, but in an attempt to solve my problem I tried loading the file with h5py instead. Unfortunately it did not work better.
Here is the error I get with h5py: "RuntimeError: Unable to get link info (bad symbol table node signature)"
The exact error may change but every time it says "bad symbol table node signature"
PS: I cannot share the code because it is huge and part of a bigger basecode that is my company's property. I can still share part of the code below to show how I load the images:
with h5py.File(dset_filepath, "r", libver='latest', swmr=True) as h5file:
    node = h5file["/train_group_0/sample_5"] # <- this line breaks
    target = node.attrs.get('TITLE').decode('utf-8')
    img = Image.fromarray(np.uint8(node))
    return img, int(target.strip())

Output:
Before accessing the dataset (node), add a test to confirm it exists. While you're adding checks, do the same for the attribute 'TITLE'. If you are going to use hard-coded path names (like 'group_0') you should check all nodes in the path exist (for example, does 'group_0' exist? Or use one of the recursive visitor functions (.visit() or .visititems() to be sure you only access existing nodes.
Modified h5py code with rudimentary checks looks like this:
sample = 'sample_5'
with h5py.File(dset_filepath, 'r', libver='latest', swmr=True) as h5file:
    if sample not in h5file['/train_group_0'].keys():
        print(f'Dataset Read Error: {sample} not found')
        return None, None
    else:
        node = h5file[f'/train_group_0/{sample}'] # <- this line breaks
        img = Image.fromarray(np.uint8(node))
        if 'TITLE' not in node.attrs.keys():
            print(f'Attribute Read Error: TITLE not found')
            return img, None
        else:
            target = node.attrs.get('TITLE').decode('utf-8')
            return img, int(target.strip())

You said you were working with PyTables. Here is code to do the same with PyTables package:
import tables as tb
sample = 'sample_5'
with tb.File(dset_filepath, 'r', libver='latest', swmr=True) as h5file:
    if sample not in h5file.get_node('/train_group_0'):
        print(f'Dataset Read Error: {sample} not found')
        return None, None
    else:
        node = h5file.get_node(f'/train_group_0/{sample}') # <- this line breaks
        img = Image.fromarray(np.uint8(node))
        if 'TITLE' not in node._v_attrs:
            print(f'Attribute Read Error: TITLE not found')
            return img, None
        else:
            target = node._v_attrs['TITLE'].decode('utf-8')
            return img, int(target.strip())


Instruction: Pytorch model gradients are printed correctly but copied wrongly

Input:
I want to copy the gradients of loss, with respect to weight, for different data samples using pytorch. In the code below, I am iterating one sample each time from the data loader (batch size = 1) and collecting gradients for 1st fully connected (fc1) layer. Gradients should be different for different samples. The print function shows correct gradients, which are different for different samples. But when I store them in a list, I get the same gradients repeatedly.   Any suggestions would be much appreciated. Thanks in advance!

grad_list = [ ]

for data in test_loader:
  inputs, labels = data[0], data[1]
  inputs = torch.autograd.Variable(inputs)
  labels = torch.autograd.Variable(labels)

  # zero the parameter gradients
  optimizer.zero_grad()

  # forward + backward
  output = target_model(inputs)
  loss = criterion(output, labels)
  loss.backward()

  grad_list.append(target_model.fc1.weight.grad.data)
  print(target_model.fc1.weight.grad.data)

Output:
Try using clone and detach instead:
grad_list.append(target_model.fc1.weight.grad.clone().detach())

The data property you are appending to your list is a mutable reference to the storage of the parameter (i.e. the actual memory address and the values contained within). What you need to do is create a replica of the gradient tensor (with clone) and remove it from the computational graph (with detach) to avoid it interfering with gradient computation.


Instruction: spectral_norm on GCNConv module

Input:
I want to call torch.nn.utils spectral_norm function on a GCNConv layer
gc1 = GCNConv(18, 16)
spectral_norm(gc1)

but I'm getting the following error:
KeyError: 'weight'

meaning gc1._parameters doesn't have weight (only bias):
gc1._parameters
OrderedDict([('bias', Parameter containing:
              tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
                     requires_grad=True))])

However, gc1.parameters() stores two objects and one of them is a 16 by 18 matrix (weight matrix).
for p in gc1.parameters():
  print('P: ', p.shape)
P:  torch.Size([16])
P:  torch.Size([16, 18])

How can I make spectral_norm function work on a GCNConv module?

Output:
According to the source code, the weight parameter is wrapped within a linear module contained in GCNConv objects as lin.
I imagine that this should then work:
gc1 = GCNConv(18, 16)
spectral_norm(gc1.lin)


Instruction: What is the equivalent of PyTorch's BoolTensor in Tensorflow 2.x?

Input:
Is there an equivalent of BoolTensor from Pytorch in Tensorflow assuming I have the below usage in Pytorch that I want to migrate to Tensorflow
done_mask = torch.BoolTensor(dones.values).to(device)
next_state_values[done_mask] = 0.0

Output:
What is dones?
Assuming it's a 0/1 tensor, you can convert it to a Bool tensor like this:
tf.cast(dones,tf.bool)

However, if you want to assign values to a tensor, you can't do it that way.
A way, which I recommend, is to multiply by a matrix of 1/0:
next_state_values *= tf.cast(dones!=1,next_state_values.dtype)

Another way , that I don't recommend as it gives some issues when using the gradient, is to use tf.tensor_scatter_nd_update. For your case, that would be:
indices = tf.where(dones==1)
next_state_values = tf.tensor_scatter_nd_update(next_state_values ,indices,2*tf.zeros(len(indices)))


Instruction: PyTorch cannot handle complex tensor on GPU, but works on CPU

Input:
I am using PyTorch to simulate NNs on a quantum computer, and therefore I have to use tensors with ComplexFloatTensor datatypes. When I run this line of code on GPU:
torch.matmul(A.transpose(1,2).flatten(0,1), H.flatten(1,2)).reshape(N,steps,2**n,2**n).transpose(0,1)

I get the following error when the tensors are LARGE:
RuntimeError: CUDA error: CUBLAS_STATUS_NOT_SUPPORTED when calling `cublasCgemm( handle, opa, opb, m, n, k, reinterpret_cast<const cuComplex*>(&alpha), reinterpret_cast<const cuComplex*>(a), lda, reinterpret_cast<const cuComplex*>(b), ldb, reinterpret_cast<const cuComplex*>(&beta), reinterpret_cast<cuComplex*>(c), ldc)`

A and H are both ComplexFloatTensor tensors.
The above error starts occurring when A and H are of shape torch.Size([100, 54, 10]) and torch.Size([54, 512, 512]) or larger, but doesn't occur when they are of shape torch.Size([100, 44, 10]) and torch.Size([44, 256, 256])
Don't worry too much about the exact numbers, but the point is that it always works on CPU (just very slowly), but on GPU it breaks past a certain size.
Does anyone know what the problem could be? Given the edit below, it might just be caused by the fact that the GPU ran out of memory (but the error failed to tell me so)
EDIT: I ran the same thing on Google Colab and got the following error at the same place:
RuntimeError: CUDA out of memory. Tried to allocate 570.00 MiB (GPU 0; 14.76 GiB total capacity; 12.19 GiB already allocated; 79.75 MiB free; 13.38 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF

Google Colab uses Tesla T4 GPUs, while my server uses NVIDIA RTX A6000

Output:
I figured out the answer to this question myself in the meantime. As it turns out, my GPU simply ran out of memory.
For some reason, Google Colab showed this error correctly (see above), while my own GPU showed this weird CUBLAS_STATUS_NOT_SUPPORTED error, instead of directly telling me that it is a memory issue.


Instruction: pytorch cuda out of memory while inferencing

Input:
I think this is a very basic question, my apologies as I am very new to pytorch. I am trying to find if an image is manipulated or not using MantraNet. After running 2-3 inferences I get the CUDA out of memory, then after restarting the kernel also I keep getting the same error: The error is given below:
RuntimeError: CUDA out of memory. Tried to allocate 616.00 MiB (GPU 0; 4.00 GiB total capacity; 1.91 GiB already allocated; 503.14 MiB free; 1.93 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
The 'tried to allocate memory(here 616.00 MiB) keeps changing. I checked the GPU statistics and it shoots up while I try to do the inferencing. In tensorflow I know we can control the memory usage by defining an upper limit, is there anything similar in pytorch that one can try?

Output: So during inferencing for a single image I realized there was no way I was able to fit my image in the gpu which I was using, so after resizing the image to a particular value I was able to do the inferencing without facing any memory issue.


Instruction: focal loss for imbalanced data using pytorch

Input:
I want to use focal loss with multiclass imbalanced data using pytorch . I searched got and try to use this code but I got  error

class_weights=tf.constant([0.21, 0.45, 0.4, 0.46, 0.48, 0.49])

loss_fn=nn.CrossEntropyLoss(weight=class_weights,reduction='mean')

and use this in train function

    preds = model(sent_id, mask, labels)

     # compu25te the validation loss between actual and predicted values
    alpha=0.25
    gamma=2
    ce_loss = loss_fn(preds, labels)
    pt = torch.exp(-ce_loss)
    focal_loss = (alpha * (1-pt)**gamma * ce_loss).mean()

the error is
TypeError: cannot assign 'tensorflow.python.framework.ops.EagerTensor' object to buffer 'weight' (torch Tensor or None required)

in this line
loss_fn=nn.CrossEntropyLoss(weight=class_weights,reduction='mean')

Output:
You're mixing tensorflow and pytorch objects.
Try:
class_weights=torch.tensor([0.21, ...], requires_grad=False)


Instruction: How to get a Pytorch data loader per class?

Input:
I want to train my model on 1 MNIST class at a time.
I can load the data with a general loader:
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.autograd import Variable

trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
# if not exist, download mnist dataset
root = './data'
train_set = datasets.MNIST(root=root, train=True, transform=trans, download=True)

batch_size = 100

train_loader = torch.utils.data.DataLoader(
                 dataset=train_set,
                 batch_size=batch_size,
                 shuffle=True)

But I'm not sure how to create 10 loaders (1 for each of the classes/digits) from this general loader (or just 10 loaders initially)

Output:
A rather simple solution would involve grouping the dataset by truth value, and creating a unique dataloader per group:
...
from torch.utils.data import Subset, DataLoader

subsets = {target: Subset(train_set, [i for i, (x, y) in enumerate(train_set) if y == target]) for _, target in train_set.class_to_idx.items()}
loaders = {target: DataLoader(subset) for target, subset in subsets.items()}

you can then pick out a specific loader based on class index:
class_3_loader = loaders[3]


Instruction: Programing a Pytorch neural network with a branch in the flow of information

Input:
I am trying to program a custom layer in PyTorch. I would like this layer to be fully connected to the previous layer but at the same time I want to feed some information from the input layer, let's say I want it to be fully connected to the first layer as well. For example the 4th layer would be fed the 3rd and 1st layer.
This would make the information flow split at the first layer and one branch would be inserted later into the network.
I have to define the forward in this layer having two inputs
class MyLayer(nn.Module):

    def __init__(self, size_in, size_out):
        super().__init__()

        self.size_in, self.size_out = size_in, size_out

        weights = torch.Tensor(size_out, size_in)

        (... ...)

    def forward(self, first_layer, previous_layer):

            (... ...)

        return output

How can I make this work if I put this layer after, let's say, a normal feed-farward which takes only the previous layer's output as input?
Can I use nn.Sequential with this layer?
Thanks!

Output:
just concatenate the input info with the output of previous layers and feed it to next layers, like:
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(100, 120) #supose your input shape is 100
        self.fc2 = nn.Linear(120, 80)
        self.fc3 = nn.Linear(180, 10)

    def forward(self, input_layer):

        x = F.relu(self.fc1(input_layer))
        x = F.relu(self.fc2(x))
        x = torch.cat((input_layer, x), 0)
        x = self.fc3(x) #this layer is fed by the input info and the previous layer
        return x


Instruction: Unable to create a custom torchtext BucketIterator

Input:
I'm trying to create a POS tagger with LSTM and I'm facing some difficulties with preparing the data.
I've successfully followed a guide that used the following code to prepare the data itertors:
TEXT = data.Field(lower = True)
UD_TAGS = data.Field(unk_token = None)
PTB_TAGS = data.Field(unk_token = None)
fields = (("text", TEXT), ("udtags", UD_TAGS), ("ptbtags", PTB_TAGS))
train_data, valid_data, test_data = datasets.UDPOS.splits(fields)
MIN_FREQ = 2

TEXT.build_vocab(train_data,
                 min_freq = MIN_FREQ,
                 vectors = "glove.6B.100d",
                 unk_init = torch.Tensor.normal_)
UD_TAGS.build_vocab(train_data)
PTB_TAGS.build_vocab(train_data)

BATCH_SIZE = 128

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    (train_data, valid_data, test_data),
    batch_size = BATCH_SIZE,
    device = device)

And then when training the model the code is:
for batch in iterator:
        text = batch.text
        tags = batch.udtags

Now for my problem - I have a dataset of lists: list of sentences (where every sentence is a list of words) and a list of lists of tags corresponsing to the sentences words.
I created a torch DataSet intance from the x_train, y_train (each one is a list of lists).
But, it does not behave like the 'train_data' that comes from datasets.UDPOS.splits(fields). So, when trying to access the data with:
for batch in iterator:
        text = batch.text
        tags = batch.udtags

I'm getting an error since my iterator does not have the fields inside. I tried accecing the data in a different manner but coudn't find a way around it.
I also noticed that in the above example, the data in the batch is with the embeddings indexes, while the batch in my code is still the words themselves.
All of the examples I found on the internet uses datasets from torchtext.legacy.datasets, so it does not really help me with my problem.
If it helps, here is my code (it's part of a bigger project, so a bit messy):
class ConvertDataset(Dataset):
    """
    Create an instances of pytorch Dataset from lists.
    """

    def __init__(self, x, y):
        # data loading
        self.x = x
        self.y = y

    def __getitem__(self, index):
        return {'text': self.x[index], 'tags': self.y[index]}

    def __len__(self):
        return len(self.x)

# ## model variables
DROPOUT = 0.25
HIDDEN_DIM = 128

# ## load and prepare train data
train_set = load_annotated_corpus(params_d['data_fn'])
x_train, y_train = _prepare_data(train_set)
TEXT = Field(lower=True)
UD_TAGS = Field(unk_token=None)

# ## build words and tags vocabularies
TEXT.build_vocab(x_train,
                 min_freq=params_d['min_frequency'],
                 vectors='glove.6B.100d',
                 unk_init=torch.Tensor.normal_,
                 max_size=None if params_d['max_vocab_size'] == -1 else
                              params_d['max_vocab_size'])

UD_TAGS.build_vocab(y_train)

# ## more model variables
INPUT_DIM = len(TEXT.vocab)
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]

# ## initiate a model
 lstm_model = BiLSTM.LSTM(input_dim=INPUT_DIM,
                             embedding_dim=params_d['embedding_dimension'],
                             hidden_dim=HIDDEN_DIM,
                             output_dim=params_d['output_dimension'],
                             n_layers=params_d['num_of_layers'],
                             dropout=DROPOUT,
                             pad_idx=PAD_IDX)

lstm_model.apply(_init_weights)

pretrained_embeddings = TEXT.vocab.vectors
lstm_model.embedding.weight.data.copy_(pretrained_embeddings)
# set pad tag embedding to 0
lstm_model.embedding.weight.data[PAD_IDX] = torch.zeros(params_d['embedding_dimension'])

BATCH_SIZE = 128

My data (lists of lists):
x_train, y_train = _prepare_data(train_data)

Data preparation
train_torch_dataset = ConvertDataset(x_train, y_train)

# ## create data iterators
train_iterator = BucketIterator(
        train_torch_dataset,
        batch_size=BATCH_SIZE,
        device=device,
        # Function to use for sorting examples.
        sort_key=lambda x: len(x['text']),
        # Repeat the iterator for multiple epochs.
        repeat=True,
        # Sort all examples in data using `sort_key`.
        sort=False,
        # Shuffle data on each epoch run.
        shuffle=True,
        # Use `sort_key` to sort examples in each batch.
        sort_within_batch=True
    )

Output:
Took me a while but I found a solution.
To create a torchtext dataset with input data as lists, use SequenceTaggingDataset (from torchtext.legacy.datasets.SequenceTaggingDataset) but you need to do a simple change to the original source code in the __init__ function, like this:
    def __init__(self, columns, fields, encoding="utf-8", separator="\t", **kwargs):
        examples = []
        # for 2 fields data sets (text, tags)
        for words, labels in zip(columns[0], columns[-1]):
            examples.append(data.Example.fromlist([words, labels], fields))
        super(SequenceTaggingDataset, self).__init__(examples, fields,
                                                     **kwargs)

Then, assuming you have a data with two field (in my example, text and pos-tags) you can define the dataset like that:
from torchtext.legacy import data

TEXT = data.Field()
UD_TAGS = data.LabelField()
# define torchtext fields
fields = (("text", TEXT), ("udtags", UD_TAGS))
# push the data into a torchtext type of dataset (** modified SequenceTaggingDataset **)
train_torchtext_dataset = SequenceTaggingDataset([x_train, y_train], fields=fields)

Note that x_train, y_train are nested lists.


Instruction: PyTorch TensorBoard add_graph() dictionary input error

Input:
What would be the proper way of passing the PyTorch dictionary dataset to the TensorBoard add_graph(model, data).
May seems similar to the Question1, Qeustion2 and Question3, however, couldn't find the right way of handling with dictionary dataset.
Error Message

Dictionary inputs to traced functions must have consistent type. Found Tensor and List[str]
Error occurs, No graph saved

Below are my anonymized scripts of the project.
train.py
from torch.utils.tensorboard import SummaryWriter
from models import CustomModel
from datasets import CustomDataset
writer = SummaryWriter()

# Dataset
dataset = CustomDataset(params ...)
train_dataset = [dataset[i] for i in range(0, k)]
train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)

# Model & TensorBoard
model = CustomModel(params....)
writer.add_graph(model, next(iter(train_dataloader)))  # ---- HERE ----

datasets.py
class CustomDataset(Dataset):
    def __init__(self, ...):
        ...
        self.x_sequences = pad_sequence(x_sequences, batch_first=True, padding_value=0)
        self.y_label = torch.LongTensor(label_list)
        ...

    def __len__(self):
        return len(self.y_label)

    def __getitem__(self, index):
        ...
        return {
            "x_categoricals": self.x_categoricals[index],
            "x_sequences": self.x_sequences[index],
            "y_label": self.y_label[index],
            "info": self.info[index],
        }

Output: The error message tells you that the entries of the dictionary must all be the same type, but in your case you seem to have a Tensor in one entry but a list of strings in another entry. You'd have to make sure that all entries have the same type.


Instruction: Switch function/class implementation between Numpy & PyTorch:?

Input:
I have a function (actually a class, but for simplicity, let's pretend it's a function) that uses several NumPy operations that all exist in PyTorch e.g. np.add and I also want a PyTorch version of the function. I'm trying to avoid duplicating my code, so I want to know:
Is there a way for me to dynamically switch a function's execution back and forth between NumPy and PyTorch without needing duplicate implementations?
For a toy example, suppose my function is:
def foo_numpy(x: np.ndarray, y: np.ndarray) -> np.ndarray:
  return np.add(x, y)

I could define a PyTorch equivalent:
def foo_torch(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
  return torch.add(x, y)

Could I somehow define a function like:
def foo(x, y, mode: str = 'numpy'):
  if mode == 'numpy':
    return np.add(x, y)
  elif mode == 'torch':
    return torch.add(x, y)
  else:
    raise ValueError

without needing the if-else statement?
Edit: what about something like the following?
def foo(x, y, mode: str = 'numpy'):
  if mode == 'numpy':
    lib = np
  elif mode == 'torch':
    lib = torch
  else:
    raise ValueError
  return lib.add(x, y)

Output:
Instead of using a string, you can use a boolean (bool) value to represent the mode you want to use i.e. False (0) representing NumPy and True (1) representing PyTorch. One can then use ternary operators to further shrink the if statements.
def foo(x, y, mode: bool = 0):
    lib = torch if mode else np
    return lib.add(x, y)

If you want to switch back and forth between the two in a class you can do something similar
class Example:

    def __init__(self):
        self._mode = True

    def switchMode(self):
        self._mode = !self._mode

    def foo(self, x, y):
        lib = torch if self._mode else np
        return lib.add(x, y)


Instruction: PyTorch equivalent of Numpy Round?

Input:
numpy.round() optionally accepts a specified number of digits to round to. However, torch.round does not, and while it seems like PyTorch will conform to NumPy eventually, what are people's current solutions?
I just want a function like torch.round(3.22, decimals=1) that returns 3.2.

Output:
You can define your own rounding function by
def round(x, decimals=0):
    b = 10**decimals
    return torch.round(x*b)/b


Instruction: Understanding Pytorch Weight and Biases in Linear Layer

Input:
Below is the code for combining weight and bias into a single layer, I am not able to understand the line below, why we have to multiply weight transpose matrix with bais. I should just bias without weight because we are multiplying weight for getting final output3
combined_layer.bias.data = layer1.bias @ layer2.weight.t() + layer2.bias
 # Create a single layer to replace the two linear layers
 combined_layer = nn.Linear(input_size, output_size)

 combined_layer.weight.data = layer2.weight @ layer1.weight
 combined_layer.bias.data = layer1.bias @ layer2.weight.t() + layer2.bias //This should be just bias

 outputs3 = inputs @ combined_layer.weight.t() + combined_layer.bias

Could anyone please help me in understanding this?

Output:
You simply need to expand the original equation of two Linear layers i.e.
# out = layer2(layer1(x))
# given (x @ A + B) @ C + D
out = (x @ layer1.weight.t() + layer1.bias) @ layer2.weight.t() + layer2.bias

You can expand (x @ A + B) @ C + D = (x @ A @ C) + B @ C + D
out = x @ layer1.weight.t() @ layer2.weight.t() + layer1.bias @ layer2.weight.t() + layer2.bias
out = x @ (layer1.weight.t() @ layer2.weight.t()) + (layer1.bias @ layer2.weight.t() + layer2.bias)
# the above equation is x @ (A @ C) + B @ C + D

# now you can assume
combined_layer.weight = layer2.weight @ layer1.weight
combined_layer.bias = layer1.bias @ layer2.weight.t() + layer2.bias
# final output
out = x @ combined_layer.weight.t() + combined_layer.bias

Also, note that matrix multiplication transpose rule is also used here i.e.

transpose(A@B) = transpose(B) @ transpose(A)

That's why combined_layer.weight.t() is multiplied by x as we didn't take transpose in layer2.weight @ layer1.weight.


Instruction: Merge one tensor into other tensor on specific indexes in PyTorch

Input:
Any efficient way to merge one tensor to another in Pytorch, but on specific indexes.
Here is my full problem.
I have a list of indexes of a tensor in below code xy is the original tensor.
I need to preserve the rows (those rows who are in indexes list) of xy  and apply some function on elements other than those indexes (For simplicity let say the function is 'multiply them with two),
xy = torch.rand(100,4)
indexes=[1,2,55,44,66,99,3,65,47,88,99,0]

Then merge them back into the original tensor.
This is what I have done so far:
I create a mask tensor
indexes=[1,2,55,44,66,99,3,65,47,88,99,0]
xy = torch.rand(100,4)
mask=[]
for i in range(0,xy.shape[0]):
    if i in indexes:
         mask.append(False)
    else:
        mask.append(True)
print(mask)
import numpy as np
target_mask = torch.from_numpy(np.array(mask, dtype=bool))
print(target_mask.sum()) #output is 89 as these are element other than preserved.

Apply the function on masked rows
zy = xy[target_mask]
print(zy)
zy=zy*2
print(zy)

Code above is working fine and posted here to clarify the problem
Now I want to merge tensor zy into xy on specified index saved in the list indexes.
Here is the pseudocode I made, as one can see it is too complex and need 3 for loops to complete the task. and it will be too much resources wastage.
# pseudocode
for masked_row in indexes:
    for xy_rows_index in xy:
        if xy_rows_index= masked_row
            pass
        else:
            take zy tensor row and replace here #another loop to read zy.

But I am not sure what is an efficient way to merge them, as I don't want to use NumPy or for loop etc. It will make the process slow, as the original tensor is too big and I am going to use GPU.
Any efficient way in Pytorch for this?

Output:
Once you have your mask you can assign updated values in place.
zy = 2 * xy[target_mask]
xy[target_mask] = zy

As for acquiring the mask I don't see a problem necessarily with your approach, though using the built-in set operations would probably be more efficient. This also gives an index tensor instead of a mask, which, depending on the number of indices being updated, may be more efficient.
i = list(set(range(len(xy)))-set(indexes))
zy = 2 * xy[i]
xy[i] = zy

Edit:
To address the comment, specifically to find the complement of indices of i we can do
i_complement = list(set(range(len(xy)))-set(i))

However, assuming indexes contains only values between 0 and len(xy)-1 then we could equivalently use i_complement = len(set(indexes)), which just removes the repeated values in indexes.


Instruction: How can I make all rows of matrix sum 1 from tensor in pytorch

Input:
I have a matrix, which is from Cora.
The size of it is [2708,1433]
For any rows, the elements are 1 or 0. I want to make "sum of elements of any rows be 1, by dividing sum of rows."
How can I make it? At first I thought I can do it by 'for' and 'append' command.
Is there any easier way?

Output:
xs = xs / xs.sum(dim=-1).unsqueeze(-1)

If xs is your Tensor, xs.sum(dim=-1) is the summation over the column-index (i.e. a Tensor of shape (2708,). By unsqueezing it, you turn it into a matrix of shape (2708, 1) which you can then broadcast against xs. The result of the division
is a matrix, all rows of which sum to 1:
xs.sum(dim=1)
assert torch.allclose(torch.ones(xs.shape[0], dtype=float), xs.sum(dim=1))

ps: if xs is ones and zeros, you might need to cast it to float first:
xs = xs.to(float)


Instruction: Detectron2 models not generating any results

Input:
I am just trying out detectron2 with some basic code as follows
model = model_zoo.get('COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml', trained=True)
im = Image.open('input.jpg')

t = transforms.ToTensor()
model.eval()

with torch.no_grad():
    im = t(im)
    output = model([{'image':im}])
    print(output)

However the model does not produce any meaningful predictions
[{'instances': Instances(num_instances=0, image_height=480, image_width=640, fields=[pred_boxes: Boxes(tensor([], device='cuda:0', size=(0, 4))), scores: tensor([], device='cuda:0'), pred_classes: tensor([], device='cuda:0', dtype=torch.int64)])}]

I don't quite get what went wrong, it was stated in the detectron2 documentation that:

You can also run inference directly like this:
model.eval()
with torch.no_grad():
   outputs = model(inputs)

and

For inference of builtin models, only “image” key is required, and “width/height” are optional.

In which case, I can't seem to find the missing link here.

Output: I had the same issue, for me I had two issues to fix.  The first was resizing shortest edge.  I used the Detectron2 built function from detectron2.data.transforms and imported ResizeShortestEdge.  The model values can be found with cfg.INPUT, which will list max/min sizes for test and train.  The other issue was matching the color channels with cfg.


Instruction: How to add two new dimensions to the input feature before the last two layers in MLP

Input:
Initially we had an MLP of multiple layers. We had an input embedding of 200 dimension. We now want to add two more dimensions to the original embedding to encode two important features. But as the original dimension is high, we fear that MLP would neglect the two new dimensions, which are quite important.
Thus, we want to add(concat) the two new dimensions before the last two layers of the MLP. I am still a new learner of ML and PyTorch, and I have searched a lot online but fail to come up with the ways to do this.
May I ask how can we achieve this using PyTorch? Thank you so much!

Output:
You could simply create two input heads. One for the embedding, which goes through its own neural network and then one for the two features. The output of both networks are then simply concatinated and passed into a final layer.
Since for the one input head there is only two features (probably a vector of size two, right?)
You can combine two neural network modules simply like this:

# create a seperate network for your embedding input
class EmbeddingModel(nn.Module):
    def __init__(self):
        super(EmbeddingModel, self).__init__()

        self.layer1 = nn.Linear(...)
        . . .
        self.layerN = nn.Linear(...)

    def forward(self, x):
        x = F.activation(self.layer1(x))
        . . .
        x = F.activation(self.layerN(x))
        return x

# create a one layer network for your "two important features"
# use the same activation function as the last layer of the "EmbeddingModel"
class FeaturesModel(nn.Module):
    def __init__(self):
        super(FeaturesModel, self).__init__()

        self.layer1 = nn.Linear(...)

    def forward(self, x):
        x = F.activation(self.layer1(x))
        return x

# finally create your main-model which combines both
class MainModel(nn.Module):
    def __init__(self):
        super(MainModel, self).__init__()

        self.embeddingModel = EmbeddingModel()
        self.featuresModel = FeaturesModel()

        # the input-dim to this layer has to be the output-dim of the embeddingModel + the output-dim of the featureModel
        self.outputLayer = nn.Linear(...)

    def forward(self, x_embeggings, x_features):
        x_embeggings = self.embeddingModel(x_embeggings)
        x_features = self.featuresModel(x_features)

        x = torch.cat((x_embeddings, x_features), -1)
        x = F.activation(self.outputLayer(x))

        return x


Instruction: How to load an ONNX file and use it to make a ML prediction in PyTorch?

Input:
Below is the source code, I use to load a .pth file and do a multi-class image classification prediction.
model = Classifier()    # The Model Class.
model.load_state_dict(torch.load('<PTH-FILE-HERE>.pth'))
model = model.to(device)
model.eval()

# prediction function to test images
def predict(img_path):
    image = Image.open(img_path)
    resize = transforms.Compose(
                    [ transforms.Resize((256,256)), transforms.ToTensor()])
    image = resize(image)
    image = image.to(device)
    y_result = model(image.unsqueeze(0))
    result_idx = y_result.argmax(dim=1)
    print(result_idx)

I converted the .pth file to an ONNX file using torch.onnx.export.
Now, How can I write a prediction script similar to above one by using the ONNX file alone and not using the .pth file.?
Is it possible to do so?

Output:
You can use ONNX Runtime.
# !pip install onnx onnxruntime-gpu
import onnx, onnxruntime

model_name = 'model.onnx'
onnx_model = onnx.load(model_name)
onnx.checker.check_model(onnx_model)

image = Image.open(img_path)
resize = transforms.Compose(
                [ transforms.Resize((256,256)), transforms.ToTensor()])
image = resize(image)
image = image.unsqueeze(0) # add fake batch dimension
image = image.to(device)

EP_list = ['CUDAExecutionProvider', 'CPUExecutionProvider']

ort_session = onnxruntime.InferenceSession(model_name, providers=EP_list)

def to_numpy(tensor):
      return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()

# compute ONNX Runtime output prediction
ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(image)}
ort_outs = ort_session.run(None, ort_inputs)

max = float('-inf')
max_index = -1
for i in range(0, len(ort_outs[0][0])):
   if(ort_outs[0][0][i] > max):
       max = ort_outs[0][0][i]
       max_index = i
print(max_index)

You can follow the tutorial for detailed explanation.
Usually, the purpose of using onnx is to load the model in a different framework and run inference there e.g. PyTorch -> ONNX -> TensorRT.


Instruction: TypeError: nll_loss_nd(): argument 'input' (position 1) must be Tensor, not tuple

Input:
So I'm trying to train my BigBird model (BigBirdForSequenceClassification) and I got to the moment of the training, which ends with below error message:
Traceback (most recent call last):
  File "C:\Users\######", line 189, in <module>
    train_loss, _ = train()
  File "C:\Users\######", line 152, in train
    loss = cross_entropy(preds, labels)
  File "C:\Users\#####\venv\lib\site-packages\torch\nn\modules\module.py", line 1102, in _call_impl
    return forward_call(*input, **kwargs)
  File "C:\Users\######\venv\lib\site-packages\torch\nn\modules\loss.py", line 211, in forward
    return F.nll_loss(input, target, weight=self.weight, ignore_index=self.ignore_index, reduction=self.reduction)
  File "C:\Users\######\venv\lib\site-packages\torch\nn\functional.py", line 2532, in nll_loss
    return torch._C._nn.nll_loss_nd(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
TypeError: nll_loss_nd(): argument 'input' (position 1) must be Tensor, not tuple

From what I understand, the problem happens because the train() function returns the tuple. Now - my question is how I should approach such issue? How do I change the output of train() function to return tensor instead of tuple?
I have seen similar issues posted here but none of the solutions seems to be helpful in my case, not even
model = BigBirdForSequenceClassification(config).from_pretrained(checkpoint, return_dict=False)

(When I don't add return_dict=False I got similiar error message but it says "TypeError: nll_loss_nd(): argument 'input' (position 1) must be Tensor, not SequenceClassifierOutput"
Please see my train code below:
def train():
    model.train()
    total_loss = 0
    total_preds = []

    for step, batch in enumerate(train_dataloader):

        if step % 10 == 0 and not step == 0:
            print('Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))

        batch = [r.to(device) for r in batch]
        sent_id, mask, labels = batch

        preds = model(sent_id, mask)

        loss = cross_entropy(preds, labels)
        total_loss = total_loss + loss.item()
        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
        optimizer.zero_grad()
        preds = preds.detach().cpu().numpy()
        total_preds.append(preds)

    avg_loss = total_loss / len(train_dataloader)
    total_preds = np.concatenate(total_preds, axis=0)
    return avg_loss, total_preds

and then:
for epoch in range(epochs):
    print('\n Epoch {:} / {:}'.format(epoch + 1, epochs))

    train_loss, _ = train()
    train_losses.append(train_loss)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

I will really appreciate any help on this case and thank you in advance!

Output: Ok, so it seems like I should have used BigBirdModel instead of BigBirdForSequenceClassification - issue solved


Instruction: Problem with Pytorch gradient of a non-sequential model

Input:
I have troubles reproducing this Pytorch tutorial.
The model they introduce is :
class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()

        self.hidden_size = hidden_size

        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
        self.i2o = nn.Linear(input_size + hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        combined = torch.cat((input, hidden), 1)
        hidden = self.i2h(combined)
        output = self.i2o(combined)
        output = self.softmax(output)
        return output, hidden

    def initHidden(self, batch_size):
        return torch.zeros(batch_size, self.hidden_size, dtype=torch.float32, requires_grad=True)

This model reproduces what is happening inside a RNN cell.
While coding, I had troubles with the gradient inside the model.
The code reproducing the issue is the following :
import torch
import torch.nn as nn

# Toy data to reproduce the issue
toy_data_batch = torch.tensor([[0, 1], [1, 0], [1, 0]], dtype=torch.float32)
toy_label_batch = torch.tensor([2, 0, 3], dtype=torch.long)

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()

        self.hidden_size = hidden_size

        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)
        self.i2o = nn.Linear(input_size + hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        combined = torch.cat((input, hidden), 1)
        hidden = self.i2h(combined)
        output = self.i2o(combined)
        output = self.softmax(output)
        return output, hidden

    def initHidden(self, batch_size):
        return torch.zeros(batch_size, self.hidden_size, dtype=torch.float32, requires_grad=True)

# Model initialization
input_size = 2
hidden_size = 2
output_size = 4  # Targets in {0, 1, 2, 3}
batch_size = 3  # 3 data points in the batch
learning_rate = 5e-3

rnn = RNN(input_size, hidden_size, output_size)
hidden = rnn.initHidden(batch_size)  # init hidden layer with zeros

# Negative log likelihood as it is classification
criterion = nn.NLLLoss()

# Forward pass
output, hidden = rnn(toy_data_batch, hidden)
#output, hidden = rnn(toy_data_batch, hidden) ### BUG: if I remove the comment here, It works

# Loss computation
loss = criterion(output, toy_label_batch)

# Backward pass
loss.backward()

print(rnn.i2o.weight.grad)  # This one is fine
print(rnn.i2h.weight.grad)  # This one isn't (has type None)

# This will fail, because of the None gradient
for weight in rnn.parameters():
    weight.data.add_(weight.grad.data, alpha=-learning_rate)

The output is :
tensor([[-0.1892,  0.0462,  0.0000,  0.0000],
        [ 0.1274,  0.1133,  0.0000,  0.0000],
        [ 0.1455, -0.2525,  0.0000,  0.0000],
        [-0.0837,  0.0930,  0.0000,  0.0000]])
None

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-3-5f02113fddf6> in <module>
     54 # This will fail, because of the None gradient
     55 for weight in rnn.parameters():
---> 56     weight.data.add_(weight.grad.data, alpha=-learning_rate)

AttributeError: 'NoneType' object has no attribute 'data'

I've noticed that if I uncomment the line
output, hidden = rnn(toy_data_batch, hidden)
#output, hidden = rnn(toy_data_batch, hidden) ### BUG: if I remove the comment here, It works

everything works without a problem. It seems to me that there is a problem with the initialization of the variable hidden. As I've turned on the argument 'requires_grad', I don't know what to do.
Thank you in advance, any help will be appreciated

Output: self.i2h has no gradient because it's not used in the first step of your model. When you back propagate, your model only uses self.i2o in the first stage, so self.i2h has no effect on the output. However, when you get to the second step, it utilises a hidden which has been calculated using self.i2h, so therefore there is a traceable gradient through that layer.


Instruction: Add(3)(5) nn.Sequential. How it works?

Input:
class Add(nn.Module):
    def __init__(self, value):
        super().__init__()
        self.value = value

    def forward(self, x):
        return x + self.value

calculator = nn.Sequential(
    Add(3),
    Add(2),
    Add(5),
)

x = torch.tensor([1])
output = calculator(x)
print(output) # tensor([11])

I made Add Model. but I can't understand how 'nn.Sequential' works.
at the firsttime, I understood like this
add = Add(torch.tensor([1]))
add(3) # tensor([4])
add = Add(add(3))
add(2) # tensor([6])
add = Add(add(2))
add(5) # tensor([11])

but Add(3)(1) also works.
I can't understand why 'Add(3)(1)' works. help me plz

Output:
You understand it right, the Sequential class in a nutshell just calls provided modules one by one. Here's the code of the forward method
    def forward(self, input):
        for module in self:
            input = module(input)
        return input

here, for module in self just iterates through the modules provided in a constructor (Sequential.__iter__ method in charge of it).
Sequential module calls this method when you call it using () syntax.
calculator = nn.Sequential(
    Add(3),
    Add(2),
    Add(5),
)
output = calculator(torch.tensor([1]))

But how does it work? In python, you could make objects of a class callable, if you add __call__ method to the class definition. If the class does not contain this method explicitly, it means that it possibly was inherited from a superclass. In case of Add and Sequential, it's Module class that implements __call__ method. And __call__ method calls 'public' forward method defined by a user.
It could be confusing that python uses the same syntax for the object instantiation and function or method call. To make a difference visible to the reader, python uses naming conventions. Classes should be named in a CamelCase with a first capital letter, and objects in a snake_case (it's not obligatory, but it's the rule that better to follow).
Just like in you example, Add is a class and add is a callable object of this class:
add = Add(torch.tensor([1]))

So, you can call add just like you have called a calculator in you example.
>>> add = Add(torch.tensor([1]))
>>> add(2)
Out: tensor([3])

But that won't work:
>>> add = Add(torch.tensor([1]))
>>> add(2)(1)
Out:
----> 3 add(2)(1)
TypeError: 'Tensor' object is not callable

That means that add(2) returns a Tensor object that does not implement __call__ method.
Compare this code with
>>> Add(torch.tensor([1]))(2)
Out:
tensor([3])

This code is the same as the first example, but rearranged a little bit.
--
To avoid confusion, I usually name objects differently: like add_obj = Add(1). It helps me to highlight a difference.
If you are not sure what you're working with, use functions type and isinstance. They would help to find out what's going on.
For example, if you check the add object, you could see that it's a callable object (i.e., it implements __call__)
>>> from typing import Callable
>>> isinstance(add, Callable)
True

And for a tensor:
>>> from typing import Callable
>>> isinstance(add, torch.tensor(1))
False

Hence, it will rase TypeError: 'Tensor' object is not callable in case you call it.
If you'd like to understand how python double-under methods like init or call work, you could read this page that describes python data model
(It could be a bit tedious, so you could prefer to read something like Fluent Python or other book)


Instruction: Pytorch is creating non empty Tensor with torch.empty((x,y)

Input:
I just want to create an empty tensor containing only zero values. but here is what I am getting so far. Source: https://pytorch.org/docs/stable/generated/torch.empty.html
a=torch.empty((2,3), dtype=torch.int32, device = 'cuda')
a

tensor([[16843009,        1,        1],
    [       0,        1,        0]], device='cuda:0', dtype=torch.int32)

Screenshot as proof:

My question is, Why?? Is it a bug or what

Output:
If you want a tensor of zeros, use torch.zeros.
torch.empty allocates a tensor but it does not initialise the contents, meaning that the tensor will contain whatever data happened to occupy that region of memory already.


Instruction: torch.no_grad() and detach() combined

Input:
I encountered many code fragments like the following for choosing an action, that include a mix of torch.no_grad and detach (where actor is some actor, SomeDistribution your preferred distribution), and I'm wondering whether they make sense:
def f():
    with torch.no_grad():
        x = actor(observation)
    dist = SomeDistribution(x)
    sample = dist.sample()
    return sample.detach()

Is the use of detach in the return statement not unnecessary, as x has its requires_grad already set to False, so all computations using x should already be detached from the graph? Or do the computations after the torch.no_grad wrapper somehow end up on the graph again, so we need to detach them once again in the end (in which case it seems to me that no_grad would be unnecessary)?
Also, if I'm right, I suppose instead of omitting detach one could also omit torch.no_grad, and end up with the same functionality, but worse performance, so torch.no_grad is to be preferred?

Output:
While it may be redundant, it depends on the internals of actor and SomeDistribution. In general, there are three cases I can think of where detach would be necessary in this code. Since you've already observed that x has requires_grad set to False then cases 2 and 3 don't apply to your specific case.

If SomeDistribution has internal parameters (leaf tensors with requires_grad=True) then dist.sample() may result in a computation graph connecting sample to those parameters. Without detaching, that computation graph, including those parameters, would be unnecessarily kept in memory after returning.
The default behavior within a torch.no_grad context is to return the result of tensor operations having requires_grad set to False. However, if actor(observation) for some reason explicitly sets requires_grad of its return value to True before returning, then a computation graph may be created that connects x to sample. Without detaching, that computation graph, including x, would be unnecessarily kept in memory after returning.
This one seems even more unlikely, but if actor(observation) actually just returns a reference to observation, and observation.requires_grad is True, then a computation graph all the way from observation to sample may be constructed during dist.sample().

As for the suggestion of removing the no_grad context in leu of detach, this may result in the construction of a computation graph connecting observation (if it requires gradients) and/or the parameters of the distribution (if it has any) to x. The graph would be discarded after detach, but it does take time and memory to create the computation graph, so there may be a performance penalty.
In conclusion, it's safer to do both no_grad and detach, though the necessity of either depends on the details of the distribution and actor.


Instruction: How can I retrieve elements in a multidimensional pytorch tensor by a list of indices?

Input:
I have two tensors: scores and lists
scores is of shape (x, 8) and lists of (x, 8, 4). I want to filter the max values for each row in scores and filter the respective elements from lists.
Take the following as an example (shape dimension 8 was reduced to 2 for simplicity):
scores = torch.tensor([[0.5, 0.4], [0.3, 0.8], ...])
lists = torch.tensor([[[0.2, 0.3, 0.1, 0.5],
                       [0.4, 0.7, 0.8, 0.2]],
                      [[0.1, 0.2, 0.1, 0.3],
                       [0.4, 0.3, 0.2, 0.5]], ...])

Then I would like to filter these tensors to:
scores = torch.tensor([0.5, 0.8, ...])
lists = torch.tensor([[0.2, 0.3, 0.1, 0.5], [0.4, 0.3, 0.2, 0.5], ...])

NOTE:
I tried so far, to retrieve the indices from the original score vector and use it as an index vector to filter lists:
# PSEUDO-CODE
indices = scores.argmax(dim=1)
for list, idx in zip(lists, indices):
    list = list[idx]

That is also where the question name is coming from.

Output:
I imagine you tried something like
indices = scores.argmax(dim=1)
selection = lists[:, indices]

This does not work because the indices are selected for every element in dimension 0, so the final shape is (x, x, 4).
The perform the correct selection you need to replace the slice with a range.
indices = scores.argmax(dim=1)
selection = lists[range(indices.size(0)), indices]


Instruction: Pytorch error when launching two distinct backward

Input:
I am building a simple autoencoder followed by an MLP neural nets. Regarging the autoencoder I am not running into any problem
# ---- Prepare training set ----
x_data = train_set_categorical.drop(["churn"], axis=1).to_numpy()
labels = train_set_categorical.loc[:, "churn"].to_numpy()

dataset = TensorDataset(torch.Tensor(x_data), torch.Tensor(labels) )

loader = DataLoader(dataset, batch_size=127)

# ---- Model Initialization ----
model = AE()

# Validation using MSE Loss function
loss_function = nn.MSELoss()

# Using an Adam Optimizer with lr = 0.1
optimizer = torch.optim.Adam(model.parameters(),
                             lr = 1e-1,
                             weight_decay = 1e-8)

epochs = 50
outputs = []
losses = []
for epoch in range(epochs):
    for (image, _) in loader:

        # Output of Autoencoder
        embbeding, reconstructed = model(image)

        # Calculating the loss function
        loss = loss_function(reconstructed, image)

        # The gradients are set to zero,
        # the the gradient is computed and stored.
        # .step() performs parameter update
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # Storing the losses in a list for plotting
        losses.append(loss)
        if epoch == 49:
            outputs.append(embbeding)

But then I am feeding an MLP with the outcome of the autoencoder and this is where things starts to fail
class Feedforward(torch.nn.Module):
    def __init__(self):
        super().__init__()

        self.neural = torch.nn.Sequential(
            torch.nn.Linear(33, 260),
            torch.nn.ReLU(),
            torch.nn.Linear(260, 450),
            torch.nn.ReLU(),
            torch.nn.Linear(450, 260),
            torch.nn.ReLU(),
            torch.nn.Linear(260, 1),
            torch.nn.Sigmoid()
        )

    def forward(self, x):
        outcome = self.neural(x.float())
        return outcome

modelz = Feedforward()
criterion = torch.nn.BCELoss()
opt = torch.optim.Adam(modelz.parameters(), lr = 0.01)

modelz.train()
epoch = 20
for epoch in range(epoch):
    opt.zero_grad()
    # Forward pass
    y_pred = modelz(x_train)
    # Compute Loss
    loss_2 = criterion(y_pred.squeeze(), torch.tensor(y_train).to(torch.float32))

    #print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
    # Backward pass
    loss_2.backward()
    opt.step()

I get the following error:
RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling .backward() or autograd.grad() the first time.

Of course I have tried to add "retain_graph=True" to both backwards or only the first one but it does not seem to solve the problem. If I launch both code independently from another It works but as a sequence I don't know why but it is not.

Output: You should be able to disconnect the output of the auto-encoder from the model by calling embbeding.detach(), before appending it to outputs.


Instruction: Why Tensorflow reports CUDA out of memory but empty_cache doesn't work?

Input:
device = torch.device("cuda:0")
model = BertModel.from_pretrained("bert-base-uncased", output_hidden_states = True)

model.to(device)

train_hidden_states = []
model.eval()

for batch in train_dataloader:
    b_input_ids = batch[0].to(device)
    b_input_mask = batch[1].to(device)

    with torch.no_grad():
        output = model(b_input_ids,
                        token_type_ids=None,
                        attention_mask=b_input_mask,
                        )
        hidden_states = output[2][12]
        train_hidden_states.append(hidden_states)

Here I am trying to get the last layer embeddings of Bert model for data in the train_dataloader.
The thing is that CUDA out of memory after 14 batches.
I tried to empty the cache, but it only decreases the GPU usage for a little bit.
with torch.cuda.device('cuda:0'):
    torch.cuda.empty_cache()

What could be the problem?

Output: You are storing tensors on GPU in train_hidden_states list. You can move then into CPU before pushing to the list train_hidden_states.append(hidden_states.cpu()).


Instruction: How to replace the value of multiple cells in multiple rows in a Pytorch tensor?

Input:
I have a tensor
import torch
torch.zeros((5,10))

>>> tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])

How can I replace the values of X random cells in each row with random inputs (torch.rand())?
That is, if X = 2, in each row, 2 random cells should be replaced with torch.rand().
Since I need it to not break backpropagation I found here that replacing the .data attribute of the cells should work.
The only familiar thing to me is using a for loop but it's not efficient for a large tensor

Output:
You can try tensor.scatter_().
x = torch.zeros(3,4)
n_replace = 3 # number of cells to be replaced with random number
src = torch.randn(x.size())
index = torch.stack([torch.randperm(x.size()[1]) for _ in range(x.size()[0])])[:,:n_replace]

x.scatter_(1, index, src)
Out[22]:
tensor([[ 0.0000,  0.5769,  0.7432, -0.1776],
        [-2.1673, -1.0802,  0.0000,  0.6241],
        [-0.6421,  0.1315,  0.0000, -2.7224]])

To avoid repetition,
perm = torch.randperm(tensor.size(0))
idx = perm[:k]
samples = tensor[idx]


Instruction: PyTorch: What's the purpose of saving the optimizer state?

Input: PyTorch is capable of saving and loading the state of an optimizer. An example is shown in the PyTorch tutorial. I'm currently just saving and loading the model state but not the optimizer. So what's the point of saving and loading the optimizer state besides not having to remember the optimizers params such as the learningrate. And what's contained in the optimizer state?

Output:
You should save the optimizer state if you want to resume model training later. Especially if Adam is your optimizer. Adam is an adaptive learning rate method, which means it computes individual learning rates for various parameters.
It is not required if you only want to use the saved model for inference.
However, It's best practice to save both model state and optimizer state.
You can also save loss history and other running metrics if you want to plot them later.
I'd do it like,
    torch.save({
            'epoch': epochs,
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict(),
            'train_loss_history': loss_history,
            }, PATH)


Instruction: Python TypeError - 'Class' object is not callable (Google Collab Example Inside)

Input:
[Redacted]
In this example, in my final cell of code, I try to call my model.  This is following the tutorial on a Youtube video.
In this step, the video is able to perform the lines
model = UCC_Classifier(config)
then in the next cell
loss, output = model(input_ids.unsqueeze(dim=0), am.unsqueeze(dim=0), labels.unsqueeze(dim=0))
To successfully get a result.  However when I try and do the same thing, I get told my class is not callable.  I cannot see any difference and am unsure why this might not be callable.
Thanks

Output: Your UCC_Classifier model should be a pl.LightningModule, not a pl.LightningDataModule.


Instruction: Replace torch.gather by other operator?

Input:
I have one script code, where x1 and x2 size of 1x68x8x8
 tmp_batch, tmp_channel, tmp_height, tmp_width = x1.size()
 x1 = x1.view(tmp_batch*tmp_channel, -1)
 max_ids = torch.argmax(x1, 1)
 max_ids = max_ids.view(-1, 1)

 x2 = x2.view(tmp_batch*tmp_channel, -1)
 outputs_x_select = torch.gather(x2, 1, max_ids) # size of 68 x 1

As for the above code, I have trouble with torch.gather when I used old onnx. Hence, I would like to find an alternative solution that replaces the toch.gather by other operators but gives the same output with the above code. Could you please give me some suggestions?

Output:
One workaround is to use the equivalent numpy method. If you include an import numpy as np statement somewhere, you could do the following.
outputs_x_select = torch.Tensor(np.take_along_axis(x2,max_ids,1))

If that gives you a grad related error, try
outputs_x_select = torch.Tensor(np.take_along_axis(x2.detach(),max_ids,1))

An approach without numpy: in this case, it seems that max_ids contains exactly one entry per row. Thus, I believe the following will work:
max_ids = torch.argmax(x1, 1) # do not reshape

x2 = x2.view(tmp_batch*tmp_channel, -1)
outputs_x_select = x2[torch.arange(tmp_batch*tmp_channel),max_ids]


Instruction: Combining Parameterlist in PyTorch

Input:
I am trying to combine two ParameterLists in Pytorch. I've implemented the following snippet:
import torch

list = nn.ParameterList()
for i in sub_list_1:
    list.append(i)
for i in sub_list_2:
    list.append(i)

Is there any functions that takes care of this without a need to loop over each list?

Output:
You can use nn.ParameterList.extend, which works like python's built-in list.extend
plist = nn.ParameterList()
plist.extend(sub_list_1)
plist.extend(sub_list_2)

Alternatively, you can use += which is just an alias for extend
plist = nn.ParameterList()
plist += sub_list_1
plist += sub_list_2


Instruction: ValueError: only one element tensors can be converted to Python scalars when converting list to float Torch tensor

Input:
I have the following:
type of X is:  <class 'list'>
X: [tensor([[1.3373, 0.5666, 0.2337,  ..., 0.4899, 0.1876, 0.5892],
        [0.0320, 0.0797, 0.0052,  ..., 0.3405, 0.0000, 0.0390],
        [0.1305, 0.1281, 0.0021,  ..., 0.6454, 0.1964, 0.0493],
        ...,
        [0.2635, 0.0237, 0.0000,  ..., 0.6635, 0.1376, 0.2988],
        [0.0241, 0.5464, 0.1263,  ..., 0.5766, 0.2352, 0.0140],
        [0.1740, 0.1664, 0.0057,  ..., 0.6056, 0.1020, 1.1573]],
       device='cuda:0')]

However, following the instructions in this video, I get this error:
    X_tensor = torch.FloatTensor(X)
ValueError: only one element tensors can be converted to Python scalars

I have the following conversion code:
X_tensor = torch.FloatTensor(X)

How can I fix this problem?
$ python
Python 3.8.10 (default, Nov 26 2021, 20:14:08)
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
'1.10.1+cu113'

X here is a list of torch.tensors.

Output:
Use torch.stack():
X = torch.stack(X)


Instruction: How do I shift specific elements of a tensor with torch.roll?

Input:
I have a tensor x, that looks like this:
x = tensor([  1,  2,  3,  4,  5],
           [  6,  7,  8,  9, 10]
           [ 11, 12, 13, 14, 15])

I'm trying to switch the first two and last two numbers of each tensor, like this:
x = tensor([  4,  5,  3,  1,  2],
           [  9, 10,  8,  6,  7],
           [ 14, 15, 13, 11, 12])

How could I do this with torch.roll()? How would I switch 3 instead of 1?

Output:
Not sure if that can be done with torch.roll alone... However, you can expect the desired result by using a temporary tensor and a pair assignment:
>>> x = torch.arange(1, 16).reshape(3,-1)
tensor([[ 1,  2,  3,  4,  5],
        [ 6,  7,  8,  9, 10],
        [11, 12, 13, 14, 15]])

>>> tmp = x.clone()

# swap the two sets of columns
>>> x[:,:2], x[:,-2:] = tmp[:,-2:], tmp[:,:2]

Such that tensor x has been mutated as:
>>> x
tensor([[ 4,  5,  3,  1,  2],
        [ 9, 10,  8,  6,  7],
        [14, 15, 13, 11, 12]])

You can pull off this operation with torch.roll and some indexing:
>>> x = torch.arange(1, 21).reshape(4,-1)
tensor([[ 1,  2,  3,  4,  5],
        [ 6,  7,  8,  9, 10],
        [11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20]])

>>> rolled = x.roll(-2,0)
tensor([[11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20],
        [ 1,  2,  3,  4,  5],
        [ 6,  7,  8,  9, 10]])

# overwrite columns [1,-1[ from rolled with those from x
>>> rolled[:, 1:-1] = x[:, 1:-1]

Such that at this end you get:
>>> rolled
tensor([[11,  2,  3,  4, 15],
        [16,  7,  8,  9, 20],
        [ 1, 12, 13, 14,  5],
        [ 6, 17, 18, 19, 10]])


Instruction: How to apply regularization only to one layer in pytorch?

Input:
Let's image a network with 2 layers (X1, X2). I want to use L1 Norm on X1 and then do (loss + L1).backward() on X1. X2 is still trained but without the regularization. My goal is to make X1 become sparse.
I have already tried this, unfortunately the regularization is applied to all layers, even though it only uses parameters from one layer.
I have also tried to freeze X1, do loss.backward() and then freeze X2 to apply do loss.backward(), including regularization. Like this:
for parameter in model.X1.parameters():
     parameter.requires_grad = False

loss.backward(retain_graph=True)

for parameter in model.X1.parameters():
     parameter.requires_grad = True
for parameter in model.X2.parameters():
     parameter.requires_grad = False

loss += l1_regularization
loss.backward()
optimizer.step()

The outcome is not as expected though. X2 does not get updated at all anymore and the values in X1  seem to be too low (all weights become very close to zero).
What am I doing wrong and is there any way to reach my goal?
Thanks for your help

Output: Your second implementation should work. However, it doesn't show the part where you set requires_grad = True for X2 afterwards (or at the start where you freeze X1). If that part is indeed missing in your code, then from the second loop onward, X2 will not get trained.


Instruction: Converting PyTorch to ONNX model increases file size for ALBert

Input:
Goal: Use this Notebook to perform quantisation on albert-base-v2 model.
Kernel: conda_pytorch_p36.

Outputs in Sections 1.2 & 2.2 show that:

converting vanilla BERT from PyTorch to ONNX stays the same size, 417.6 MB.
Quantization models are smaller than vanilla BERT, PyTorch 173.0 MB and ONNX 104.8 MB.

However, when running ALBert:

PyTorch and ONNX model sizes are different.
Quantized model sizes are bigger than vanilla.

I think this is the reason for poorer model performance of both Quantization methods of ALBert, compared to vanilla ALBert.
PyTorch:
Size (MB): 44.58906650543213
Size (MB): 22.373255729675293

ONNX:
ONNX full precision model size (MB): 341.64233207702637
ONNX quantized model size (MB): 85.53886985778809

Why might exporting ALBert from PyTorch to ONNX increase model size, but not for BERT?
Please let me know if there's anything else I can add to post.

Output:
Explanation
ALBert model has shared weights among layers. torch.onnx.export outputs the weights to different tensors, which causes the model size to grow larger.
A number of Git Issues have been marked Solved regarding this phenomena.
The most common solution is to remove shared weights, that is to remove tensor arrays that contain the exact same values.

Solutions
Section "Removing shared weights" in onnx_remove_shared_weights.ipynb.
Pseudo-code:
from onnxruntime.transformers.onnx_model import OnnxModel
model=onnx.load(path)
onnx_model=OnnxModel(model)
count = len(model.graph.initializer)
same = [-1] * count
for i in range(count - 1):
  if same[i] >= 0:
    continue
  for j in range(i+1, count):
     if has_same_value(model.graph.initializer[i], model.graph.initializer[j]):
    same[j] = i

for i in range(count):
   if same[i] >= 0:
        onnx_model.replace_input_of_all_nodes(model.graph.initializer[i].name, model.graph.initializer[same[i]].name)

onnx_model.update_graph()
onnx_model.save_model_to_file(output_path)

Source of both solutions


Instruction: Incorporating dim of torch.topk in tf.nn.top_k

Input:
Pytorch provide torch.topk(input, k, dim=None, largest=True, sorted=True) function to calculate k largest elements of the given input tensor along a given dimension dim.
I have a tensor of shape (64, 128, 512) and I am using torch.topk in the following manner-
reduce = input.topk(k, dim=1).values

I found similar tensorflow implementaion as following - tf.nn.top_k(input, k=1, sorted=True, name=None).
My question is how to incorporate dim=1 parameter in tf.nn.top_k so as to achieve the tensor of the same shape as calculated by pytorch?

Output:
I agree with @jodag, you will have to transpose or reshape your tensor, since tf.math.top_k always works on the last dimension.
What you could also do is first get all the max values in the tensor along a certain dimension and then get the top k values from that tensor:
import tensorflow as tf
tf.random.set_seed(2)

k = 3
tensor = tf.random.uniform((2, 4, 6), maxval=10, dtype=tf.int32)
max_tensor = tf.reduce_max(tensor, axis=1)
k_max_tensor = tf.math.top_k(max_tensor, k=k, sorted=True).values

print('Original tensor --> ', tensor)
print('Max tensor --> ', max_tensor)
print('K-Max tensor --> ', k_max_tensor)
print('Unique K-Max tensor', tf.unique(tf.reshape(k_max_tensor, (tf.math.reduce_prod(tf.shape(k_max_tensor)), ))).y)

Original tensor -->  tf.Tensor(
[[[1 6 2 7 3 6]
  [7 5 1 1 0 6]
  [9 1 3 9 1 4]
  [6 0 6 2 4 0]]

 [[4 6 8 2 4 7]
  [5 0 8 2 8 9]
  [0 2 0 0 9 8]
  [9 3 8 9 0 6]]], shape=(2, 4, 6), dtype=int32)
Max tensor -->  tf.Tensor(
[[9 6 6 9 4 6]
 [9 6 8 9 9 9]], shape=(2, 6), dtype=int32)
K-Max tensor -->  tf.Tensor(
[[9 9 6]
 [9 9 9]], shape=(2, 3), dtype=int32)
Unique K-Max tensor tf.Tensor([9 6], shape=(2,), dtype=int32)


Instruction: pytorch lightning epoch_end/validation_epoch_end

Input:
Could anybody breakdown the code and explain it to me? The part that needs help is indicated with the "#This part". I would greatly appreciate any help thanks
def validation_epoch_end(self, outputs):
    batch_losses = [x["val_loss"]for x in outputs] #This part
    epoch_loss = torch.stack(batch_losses).mean()
    batch_accs =  [x["val_acc"]for x in outputs]   #This part
    epoch_acc = torch.stack(batch_accs).mean()
    return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}

def epoch_end(self, epoch, result):
    print("Epoch [{}], val_loss: {:.4f}, val_acc: {:.4f}".format( epoch,result['val_loss'], result['val_acc'])) #This part

Output:
In your provided snippet, outputs is a list containing dicts elements which seem to contain at least keys "val_loss", and "val_acc". It would be fair to assume they correspond to the validation loss and validation accuracy respectively.
Those two lines (annotated with the # This path comment) correspond to list comprehensions going over the elements inside the outputs list. The first one gathers the values of the key "val_loss" for each element in outputs. The second one does the same this time gathering the values of the "val_acc" key.
A minimal example would be:
## before
outputs = [{'val_loss': tensor(a), # element 0
            'val_acc': tensor(b)},

           {'val_loss': tensor(c), # element 1
            'val_acc': tensor(d)}]

## after
batch_losses = [tensor(a), tensor(c)]
batch_acc = [tensor(b), tensor(d)]


Instruction: Is this the right way to compute cosine similarity in PyTroch?

Input:
cos = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
c = torch.FloatTensor([1, 2, 4])
b = torch.FloatTensor([1, 2, 3])
simi = cos(b,c)
tensor(0.9915)

I am using dim=-1 in this funciton, does that mean it is a one dimension float list? Is this correct?

Output:
Like with most indexing in python, -1 refers to last dimension (-2 would be second-to-last, etc...). Using dim=-1 when initializing cosine similarity means that cosine similarity will be computed along the last dimension of the inputs.
For example, if b and c were 3-dimensional tensors with size [X,Y,Z], then the result would be a 2-dimensional tensor of size [X,Y]. In your case, since the input tensors only have one dimension (size [3]), you end up getting a result tensor of size [], i.e. a scalar.


Instruction: Torchscript trace "must be on the current device" error despite model and input both being on the same device

Input:
I am failing to run torch.jit.trace despite my best effort, encountering RuntimeError: Input, output and indices must be on the current device
I have a (fairly complex) model which I have already put on GPU, along with a set of inputs, also on GPU. I can verify that all input tensors and model parameters & buffers are on the same device:
(Pdb) {p.device for p in self.parameters()}
{device(type='cuda', index=0)}
(Pdb) {p.device for p in self.buffers()}
{device(type='cuda', index=0)}
(Pdb) in_ = (<several tensors here>)
(Pdb) {p.device for p in in_}
{device(type='cuda', index=0)}
(Pdb) torch.cuda.current_device()
0

I can certify the model runs and the output is on the correct device:
(Pdb) self(*in_).device
device(type='cuda', index=0)

Despite all this, tracing fails:
(Pdb) generator_script = torch.jit.trace(self, example_inputs=in_)
*** RuntimeError: Input, output and indices must be on the current device

I understand about inputs and outputs, but what are these "indices"
that must also be on the same device?
What other elements that I am
not accounting for could be causing trace to fail?

Output:
If you're not yet mapping the device during the loading process, doing so could be the solution.[1] That is, mapping the device should happen during jit.load, not as a simple call of .to(device) after jit.load has already finished. See this page for more info.
As an example of what to do:
model = jit.load("your_traced_model.pt", map_location=torch.device("cuda"))

This is different from how it works for typical/non-JIT models, where you can simply do:
model = some_model_creation_function()
_ = model.to(torch.device("cuda"))

1 = this does not currently work for the MPS device.


Instruction: Creating a new tensor according to a list of lengths

Input:
I have a tensor t with dim b x 3 and a list of lengths len = [l_0, l_1, ..., l_n]. All entries in len sum to to b. I want to create a new tensor with dim n x 3, which stores the average of the entries in t. E.g. The first l_0 entries in t are averaged and build the first element in the new tensor. The following l_1 entries are averaged and build the second element, ...
Thanks for your help.

Output:
You can do so using a combination a cumulative list of indices as helper and a list comprehension to construct the new array:
>>> b, lens = 10, [2, 3, 1, 3, 1]

>>> t =  torch.rand(b, 3)
tensor([[0.3567, 0.3998, 0.9396],
        [0.4061, 0.6465, 0.6955],
        [0.3500, 0.4135, 0.5288],
        [0.0726, 0.9575, 0.3785],
        [0.6216, 0.2975, 0.3293],
        [0.3878, 0.0735, 0.8181],
        [0.1694, 0.5446, 0.1179],
        [0.7793, 0.6613, 0.1748],
        [0.0964, 0.9825, 0.1651],
        [0.1421, 0.0994, 0.8086]])

Build the cumulative list of indices:
>>> c = torch.cumsum(torch.tensor([0] + lens), 0)
tensor([ 0,  2,  5,  6,  9, 10])

Loop over c by twos, with an overlapping window. For example zip(c[:-1], c[1:]) works well. Each selection from i to j gets averaged on dim=0.
>>> [t[i:j].sum(0) for i, j in zip(c[:-1], c[1:])]
[tensor([0.7628, 1.0463, 1.6351]),
 tensor([1.0442, 1.6685, 1.2367]),
 tensor([0.3878, 0.0735, 0.8181]),
 tensor([1.0451, 2.1885, 0.4578]),
 tensor([0.1421, 0.0994, 0.8086])]

Then you can stack the list:
>>> torch.stack([t[i:j].sum(0) for i, j in zip(c[:-1], c[1:])])
tensor([[0.7628, 1.0463, 1.6351],
        [1.0442, 1.6685, 1.2367],
        [0.3878, 0.0735, 0.8181],
        [1.0451, 2.1885, 0.4578],
        [0.1421, 0.0994, 0.8086]])


Instruction: Pytorch Autograd with complex fourier transforms gives wrong results

Input:
I am trying to implement a real valued cost function that evaluates a complex input in frequency space with pytorch & autograd since I am interested in the gradients of the cost function w.r.t. the input. When I compare the autograd results with the derivative that I computed by hand (with Wirtinger calculus) I get a different result. I'm not sure where I made the mistake, whether it is in my implementation or in my own derivation of the gradient.
The cost function and its derivative by hand looks like this:
Formula of the cost function
My implementation is here
def f_derivative_by_hand(f):
    f = torch.tensor(f, dtype=torch.complex128)
    ftilde = torch.fft.fft(f)
    absf = torch.abs(ftilde)
    f2 = absf**2
    C = torch.trapz(f2).numpy()
    grads = 2 * torch.fft.ifft((ftilde)).numpy()
    return C, grads

def f_derivative_autograd(f):
    f = torch.tensor(f, dtype=torch.complex128, requires_grad=True)
    ftilde = torch.fft.fft(f)
    f2 = torch.abs(ftilde)**2
    C = torch.trapz(f2)
    C.backward()
    grads = f.grad
    return C.detach().numpy(), grads.detach().numpy()

When I use some data and evaluate it by both functions, the gradients of the implementation with automatic differentiation is tilted in comparison (note that I normalized the plotted arrays):
Autograd and derivative by hand comparison
I suspect there could also be something wrong with the automatic differentiation of fft though since if I remove the fourier transform from the cost function and integrate the function in real space, both implementations match exactly except at the edges (again normalized):
No FFT autograd and derivative by hand
It would be fantasic if someone could help me figure out what is wrong!

Output:
After some more investigation, I found the solution to the problem of the tilted derivatives. Apparently, the trapezoidal integration rule assumes boundary conditions that will show some artifacts at the boundaries as discussed in this pytorch forum post.
In my original problem, the observed tilt results from the integration of the fourier transformed signal which is asymmetric. The boundary artifacts introduce spatial frequencies which tilt the derivative in real space.
For me, the simplest solution is just to use a sum and weight by the frequency differential. Then, everything works out.


Instruction: RuntimeError: Expected all tensors to be on the same device

Input:
I am getting the following error:
RuntimeError: Expected all tensors to be on the same device

However, both my tensor are using .to(device=t.device).
            self.indices_buf = torch.LongTensor().to(device=t.device)
            self.beams_buf = torch.LongTensor().to(device=t.device)
            self.beams_buf_float = torch.FloatTensor().to(device=t.device)

Here self.beams_buf_float.type(torch.LongTensor) gives Expected all tensors to be on the same device error.
        torch.div(self.indices_buf, vocab_size, out=self.beams_buf_float)
        self.beams_buf = self.beams_buf_float.type(torch.LongTensor)

I am confused here, as all of them are using device=t.device.

Output:
When calling self.beams_buf_float.type(torch.LongTensor), the resulting tensor  device is set to the default one (i.e. cpu).
The correct way to cast your tensor to a new type while maintaining the original device is by calling self.brams_buf_float.to(torch.long) or self.brams_buf_float.long()


Instruction: where can I find the source code for torch.unique()?

Input:
I can only find in the pytorch source code (https://github.com/pytorch/pytorch/blob/2367face24afb159f73ebf40dc6f23e46132b770/torch/functional.py#L783) the following function call:
_VF.unique_dim() and torch._unique2()
but they don't point to anywhere else in the directory

Output:
Most of the pytorch backend code is implemented in C++ and/or CUDA. To see it you need to find the appropriate entrypoint in the source code. There are a couple ways to do this but the easiest I've found without downloading all the code yourself is to search for the keywords on github.
For example, if you go to github.com and the search for unique_dim repo:pytorch/pytorch, then click the "Code" tab on the left side you should quickly find the following.
From torch/jit/_builtins.py:103
 17: _builtin_ops = [
...
103:    (torch._VF.unique_dim, "aten::unique_dim"),

From this and further analysis of the code we can conclude that torch._VF.unique_dim is actually invoking the aten::unique_dim function from the ATen library.
Like most functions in ATen there are multiple implementations of this function. Most ATen functions are registered in aten/src/ATen/native/native_functions.yaml, generally the functions here will have a _cpu and _cuda version.
Going back to the search results we can find that the CUDA implementation is actually calling the function unique_dim_cuda at aten/src/ATen/native/cuda/Unique.cu:197
196: std::tuple<Tensor, Tensor, Tensor>
197: unique_dim_cuda(const Tensor& self, const int64_t dim, const bool sorted, const bool return_inverse, const bool return_counts) {
198:   return AT_DISPATCH_ALL_TYPES_AND2(kBool, kHalf, self.scalar_type(), "unique_dim", [&] {
199:     return unique_dim_cuda_template<scalar_t>(self, dim, false, return_inverse, return_counts);
200:   });
201: }

and the CPU implementation is calling the function unique_dim_cpu at aten/src/ATen/native/Unique.cpp:271
270: std::tuple<Tensor, Tensor, Tensor>
271: unique_dim_cpu(const Tensor& self, const int64_t dim, const bool sorted, const bool return_inverse, const bool return_counts) {
272:   return AT_DISPATCH_ALL_TYPES_AND2(at::ScalarType::BFloat16, at::ScalarType::Bool, self.scalar_type(), "unique_dim", [&] {
273:     // The current implementation using `dim` always sorts due to unhashable tensors
274:     return _unique_dim_cpu_template<scalar_t>(self, dim, false, return_inverse, return_counts);
275:   });
276: }

From this point you should be able to trace the function calls further down to see exactly what they are doing.
Following a similar string of searches you should find that torch._unique2 is implemented at aten/src/ATen/native/cuda/Unique.cu:188 and aten/src/ATen/native/Unique.cpp:264 for CUDA and CPU respectively.


Instruction: Argmax of 2d vector on C++

Input:
I am working on python/pytorch and I have an example like
2d vector a

    |
    v
  dim-0  ---> -----> dim-1 ------> -----> --------> dim-1
    |   [[-1.7739,  0.8073,  0.0472, -0.4084],
    v    [ 0.6378,  0.6575, -1.2970, -0.0625],
    |    [ 1.7970, -1.3463,  0.9011, -0.8704],
    v    [ 1.5639,  0.7123,  0.0385,  1.8410]]
    |
    v

Then, the argmax with the index of 1 will be
# argmax (indices where max values are present) along dimension-1
In [215]: torch.argmax(a, dim=1)
Out[215]: tensor([1, 1, 0, 3])

My question is that given the 2d vector a as above, how could I implement argmax function on C++ to give me same output as above? Thanks for reading
This is what I did

    vector<vector<float>> a_vect

    {
      {-1.7739,  0.8073,  0.0472, -0.4084},
      {0.6378,  0.6575, -1.2970, -0.0625},
      {1.7970, -1.3463,  0.9011, -0.8704},
      {1.5639,  0.7123,  0.0385,  1.8410}
     };
     std::vector<int>::iterator max = max_element(a_vect.begin() , a_vect.end()-a_vect.begin());

Output:
You can use std::max_element to find the index in each sub vector
#include <algorithm>
#include <iostream>
#include <vector>
using std::vector;

int main()
{
  vector<vector<float>> a_vect=
  {
    {-1.7739,  0.8073,  0.0472, -0.4084},
    {0.6378,  0.6575, -1.2970, -0.0625},
    {1.7970, -1.3463,  0.9011, -0.8704},
    {1.5639,  0.7123,  0.0385,  1.8410}
  };

  vector<int> max_index;
  for(auto& v:a_vect)
    max_index.push_back(std::max_element(v.begin(),v.end())-v.begin());

  for(auto i:max_index)
    std::cout << i << ' '; // 1 1 0 3
}


Instruction: Alternate training between two datasets

Input:
I am trying to use augmented and not augmented datasets in each epoch(for example: augmented in one epoch not augmented in a different epoch) but I couldn't figure out how to do it. My approach was loading DataLoader in each epoch again and again but I think it's wrong. Because when I print indexes in __getitem__ in my Dataset class, there is a lot of duplicates indices.
Here is my code for training:
for i in range(epoch):

    train_loss = 0.0
    valid_loss = 0.0
    since = time.time()
    scheduler.step(i)
    lr = scheduler.get_lr()

    #######################################################
    #Training Data
    #######################################################

    model_test.train()
    k = 1
    tx=""
    lx=""
    random_ = random.randint(0,1)
    print("QPQPQPQPQPQPQPQPPQPQ")
    print(random_)
    print("QPQPQPQPQPQPQPQPPQPQ")
    if random_== 0:
            tx = torchvision.transforms.Compose([
                #  torchvision.transforms.Resize((128,128)),
                torchvision.transforms.ToTensor(),
                torchvision.transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])

            lx = torchvision.transforms.Compose([
                    #  torchvision.transforms.Resize((128,128)),
                    torchvision.transforms.Grayscale(),
                    torchvision.transforms.ToTensor(),
                    # torchvision.transforms.Lambda(lambda x: torch.cat([x, 1 - x], dim=0))
                ])
    else:
            tx = torchvision.transforms.Compose([
                #  torchvision.transforms.Resize((128,128)),

                torchvision.transforms.CenterCrop(96),
                torchvision.transforms.RandomRotation((-10, 10)),
                # torchvision.transforms.RandomHorizontalFlip(),
                torchvision.transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
                torchvision.transforms.ToTensor(),
                torchvision.transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
            ])
            lx = torchvision.transforms.Compose([
                    #  torchvision.transforms.Resize((128,128)),

                    torchvision.transforms.CenterCrop(96),
                    torchvision.transforms.RandomRotation((-10, 10)),
                    torchvision.transforms.Grayscale(),
                    torchvision.transforms.ToTensor(),
                    # torchvision.transforms.Lambda(lambda x: torch.cat([x, 1 - x], dim=0))
                ])
    Training_Data = Images_Dataset_folder(t_data,
                                      l_data,tx,lx)
    train_loader = torch.utils.data.DataLoader(Training_Data, batch_size=batch_size, sampler=train_sampler,
                                           num_workers=num_workers, pin_memory=pin_memory,)

    valid_loader = torch.utils.data.DataLoader(Training_Data, batch_size=batch_size, sampler=valid_sampler,
                                           num_workers=num_workers, pin_memory=pin_memory,)

    for x,y in train_loader:

        x, y = x.to(device), y.to(device)

        #If want to get the input images with their Augmentation - To check the data flowing in net
        input_images(x, y, i, n_iter, k)

       # grid_img = torchvision.utils.make_grid(x)
        #writer1.add_image('images', grid_img, 0)

       # grid_lab = torchvision.utils.make_grid(y)

        opt.zero_grad()

        y_pred = model_test(x)
        lossT = calc_loss(y_pred, y)     # Dice_loss Used

        train_loss += lossT.item() * x.size(0)
        lossT.backward()
      #  plot_grad_flow(model_test.named_parameters(), n_iter)
        opt.step()
        x_size = lossT.item() * x.size(0)
        k = 2

Here is my code for the dataset:
    def __init__(self, images_dir, labels_dir, transformI=None,
        transformM=None):
        self.images = sorted(os.listdir(images_dir))
        self.labels = sorted(os.listdir(labels_dir))
        self.images_dir = images_dir
        self.labels_dir = labels_dir
        self.transformI = transformI
        self.transformM = transformM
        self.tx=self.transformI
        self.lx=self.transformM

    def __len__(self):

        return len(self.images)

    def __getitem__(self, i):

        with open("/content/x.txt", "a") as o:
            o.write(str(i)+"\n")
        i1 = Image.open(self.images_dir + self.images[i])
        l1 = Image.open(self.labels_dir + self.labels[i])

        seed = np.random.randint(0, 2 ** 32)  # make a seed with numpy generator

        # apply this seed to img tranfsorms
        random.seed(seed)
        torch.manual_seed(seed)

        img = self.tx(i1)

        # apply this seed to target/label tranfsorms
        random.seed(seed)
        torch.manual_seed(seed)
        label = self.lx(l1)

        return img, label

How can i achieve what i want?
Thanks in advance.

Output:
Instantiating a dataset and data loader for each epoch doesn't seem to be the way to go. Instead, you may want to instantiate two sets of dataset + data loader, each one with its corresponding augmentation pipeline.
Here is an example to give you a basic frame:
Start by defining the transformation pipelines inside of the dataset itself:
class Images_Dataset_folder(Dataset):
    def __init__(self, images_dir, labels_dir, augment=False):
        super().__init__()
        self.tx, self.lx = self._augmentations() if augment else self._no_augmentations()

    def __len__(self):
        pass

    def __getitem__(self, i):
        pass

    def _augmentations(self):
        tx = T.Compose([
            T.ToTensor(),
            T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])

        lx = T.Compose([
            T.Grayscale(),
            T.ToTensor()])

        return tx, lx

    def _no_augmentations(self):
        tx = T.Compose([
                T.CenterCrop(96),
                T.RandomRotation((-10, 10)),
                T.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),
                T.ToTensor(),
                T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])

        lx = T.Compose([
            T.CenterCrop(96),
            T.RandomRotation((-10, 10)),
            T.Grayscale(),
            T.ToTensor()])

        return tx, lx

Then you can construct your training loop as:
# augmented images dataset
aug_trainset = Images_Dataset_folder(t_data, l_data, augment=True)
aug_dataloader= DataLoader(aug_trainset, batch_size=batch_size)

# unaugmented images dataset
unaug_trainset = Images_Dataset_folder(t_data, l_data, augment=False)
unaug_dataloader = DataLoader(unaug_trainset, batch_size=batch_size)

# on each epoch you go through the
for i in range(epochs//2):
    # call train loop on augmented data loader
    train(model, aug_dataloader)

    # call train loop with un-augmented data loader
    train(model, unaug_dataloader )

This being said, you will essentially loop over the dataset twice: once on unaugmented images and a second time around with augmented images.
If you want to only iterate only once, then the easiest solution I can come up with is having a random flag inside the __getitem__ that would decide whether or not the current image needs to get augmented.

Side note: you wouldn't want to use train data in your validation set!


Instruction: Element-wise matrix vector multiplication

Input:
I have a tensor m which stores n 3 x 3 matrices with dim n x 3 x 3 and a tensor v with n 3x1 vectors and dim n x 3. How can I apply element-wise matrix-vector multiplication, i.e. multiply the i-th matrix with the i-th vector, to get an output tensor with dim n x 3?
Thanks for your help.

Output:
You want to perform a matrix multiplication operation (__matmul__) in a batch-wise manner. Intuitively you can use the batch-matmul operator torch.bmm. Keep in mind you first need to unsqueeze one dimension on v such that it becomes a 3D tensor. In this case indexing the last dimension with None as v[..., None] will provide a shape of (n, 3, 1).
With torch.bmm:
>>> torch.bmm(m, v[..., None])

As it turns out, torch.matmul handles this case out-of-the-box:
>>> torch.matmul(m, v[..., None]) # same as m@v[..., None]

If you want explicit control over the operation, you can go with torch.einsum:
>>> torch.einsum('bij,bj->bi', m, v)