Instruction: How to implement Laplace Posteriori Approximation on BERT in PyTorch?

Input:
I'm trying to implement the Laplace Posteriori Approximation on the last layer for the classification results obtained by BERT model. I get an error regarding input size, and after I fix it by extracting just embeddings and class labels from BERT to feed them into Laplace, I get another bunch of errors regarding input dimensions that I don't know how to debug.
As this is something I didn't find on the internet, and includes relatively new libraries, I will post here just the first error I got, code that might help in debugging and useful links.
I will update post if needed.
Of course, if someone knows how to implement Laplace Posteriori Approximation with BERT in some other library like Scikit or Trax, it would be helpful. Also, some other Transformer classification model with some other confidence approximation will be useful for me. Any help is appreciated!
Code:
# Import
import pandas as pd
import torch
from torch.utils.data import DataLoader
from torch.utils.data import TensorDataset
from torch import nn
from transformers import BertTokenizer
from transformers import BertModel
from transformers import BertForSequenceClassification
from sklearn.model_selection import train_test_split
import time
import os

#Toy Data
data_a_b_c = ["""category a. This is category a. In category a we talk about animals.
This category includes lions, fish, tigers, birds, elephants, mouses, dogs, cats, and all other animals."""] * 60 \
+  ["""category b. This is category b. In category b we talk about people. This category members are
Abraham Maslow, John Lennon, Drazen Petrovic, Nikola Tesla, Slavoljub Penkala, Nenad Bakic and Larry Page."""] * 60 \
+ ["""category c. This is category c. Category c is dedicated to car brands like Lamborgini, Rimac-Buggati, BMW, Mercedes,
Honda, Opel, Wolkswagen, and etc."""] * 60

label_0_1_2 =  [0] * 60 + [1] * 60 + [2] * 60
d = {'text': data_a_b_c, 'labels': label_0_1_2}
df = pd.DataFrame(data=d)

print(df.head(3))
print(df.tail(3))
print(df.info())

# Parameters
tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
batch_size = 2
learning_rate = 3e-4
epochs = 3
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
labels = pd.Series(df.labels.values).to_dict()
num_classes = 3

print(f'Tokenizer: {tokenizer}, Batch size:{batch_size}, Learning rate:{learning_rate}, Epochs:{epochs}')
print('Device: ', device)
print('Number of possible classes: ', num_classes)

# Model Architecture
class TransformerModel(nn.Module):
    def __init__(self, num_classes, dropout=0.5):
        super(TransformerModel, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-multilingual-cased')
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, num_classes)
        self.relu = nn.ReLU()

    def forward(self, input_id, mask):
        _, pooled_output = self.bert(input_ids=input_id, attention_mask=mask, return_dict=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        final_layer = self.relu(linear_output)

        return final_layer

# Prepare Data Function
def prepare_data(data, labels):
    texts = tokenizer(data, padding='max_length', max_length=512, truncation=True, return_tensors="pt")
    input_ids = texts['input_ids']
    attention_mask = texts['attention_mask']
    train_dataset = TensorDataset(input_ids, attention_mask, torch.LongTensor(labels))
    dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)

    return dataloader

#Run Training Function
def run_training(train_dataloader, val_dataloader, epochs=epochs, lr=learning_rate):

    def train(dataloader):
        model.train()
        total_acc, total_count = 0, 0
        log_interval = 128
        start_time = time.time()

        for idx, (input_id, mask, label) in enumerate(train_dataloader):
           # print(idx)
            mask = mask.to(device)
            input_id = input_id.to(device)
            label = label.type(torch.LongTensor).to(device)

            output = model(input_id, mask)

            optimizer.zero_grad()
            loss = criterion(output, label)
            loss.backward()
           # torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
            optimizer.step()

            total_acc += (output.argmax(1) == label).sum().item()
            total_count += label.size(0)

            if idx % log_interval == 0 and idx > 0:
                elapsed = time.time() - start_time
                print('| epoch {:3d} | {:5d}/{:5d} batches '
                      '| accuracy {:8.3f}'.format(epoch, idx, len(dataloader),
                                                  total_acc / total_count))
                total_acc, total_count = 0, 0
                start_time = time.time()

    def evaluate(dataloader):
        model.eval()
        total_acc, total_count = 0, 0
        with torch.no_grad():
              for idx, (input_id, mask, label) in enumerate(dataloader):
                mask = mask.to(device)
                input_id = input_id.to(device)
                label = label.to(device)
                output = model(input_id, mask)
                total_acc += (output.argmax(1) == label).sum().item()
                total_count += label.size(0)
        return total_acc / total_count

    criterion = torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.SGD(model.parameters(), lr=lr)
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.1)

    cuda = torch.cuda.is_available()
    device = torch.device("cuda" if cuda else "cpu")
    device = 'cuda'
    model.to(device)

    total_accu = None
    for epoch in range(1, epochs + 1):
        epoch_start_time = time.time()
        train(train_dataloader)
        accu_val = evaluate(val_dataloader)
        if total_accu is not None and total_accu > accu_val:
            scheduler.step()
        else:
            total_accu = accu_val
        print('-' * 59)
        print('| end of epoch {:3d} | time: {:5.2f}s | '
              'valid accuracy {:8.3f} '.format(epoch,
                                               time.time() - epoch_start_time,
                                               accu_val))
        print('-' * 59)

# Data Split And Preparation
X_train, X_test, y_train, y_test = train_test_split(df.text.values.tolist(), df.labels.values.tolist(), test_size=0.2, random_state=2)
train_dataloader = prepare_data(X_train, y_train)
val_dataloader = prepare_data(X_test, y_test)

# Run The Model
model = TransformerModel(num_classes)
run_training(train_dataloader, val_dataloader)
print('finished')

# Save And Load The Model (if needed)
PATH = ".../Torch_BERT_model"
torch.save(model, os.path.join(PATH, "Toy_Data_BERT.pth"))
model = torch.load(os.path.join(PATH, "Toy_Data_BERT.pth"))
print(model)

# Laplace
from laplace import Laplace
la = Laplace(model, 'classification', subset_of_weights='last_layer', hessian_structure='full')
la.fit(train_dataloader)

Error I get:

--------------------------------------------------------------------------- ValueError                                Traceback (most recent call
last) ~\AppData\Local\Temp\ipykernel_7144\3779742208.py in <cell line:
2>()
1 la = Laplace(model, 'classification', subset_of_weights='last_layer', hessian_structure='full')
----> 2 la.fit(train_dataloader)
~\anaconda3\lib\site-packages\laplace\lllaplace.py in fit(self,
train_loader, override)
98
99         if self.model.last_layer is None:
--> 100             X, _ = next(iter(train_loader))
101             with torch.no_grad():
102                 try:
ValueError: too many values to unpack (expected 2)

Useful link for Laplace implementation with examples:
https://aleximmer.github.io/Laplace/#full-example-optimization-of-the-marginal-likelihood-and-prediction
Code that might help in debugging:
for x in train_dataloader:
        print("The length of batch is:", len(x))
        print()
        print("The batch looks like:", x)
        print()

        print("The length of the first element in the batch is:") #embedding
        print(len(x[0]))
        print("The length of the second element in the batch is:") #1 if place is filled with word, 0 if it's empty?
        print(len(x[1]))
        print("The length of the third element in the batch is:") #category
        print(len(x[2]))
        print()

        print("The lengths of the first tensor and second tensor in the first element in the batch is:")
        print(len(x[0][0]), len(x[0][1])) # = max_length (512)
        print("The lengths of the first tensor and second tensor in the second element in the batch is:")
        print(len(x[1][0]), len(x[1][1])) # = max_length (512)
        print()
        print()

Output:
The laplace library expects that the dataloader returns two parameters (X,y) and that the model requires exactly one argument to make its prediction (code). But your model forward pass requires two arguments, namely input_id and mask, and your dataloader returns three arguments input_id, mask, and labels.
There are several ways to work around this limitation (e.g. return a dict with input_ids and attention_mask). The way that requires the least understanding of the internals of the laplace library is to generate the attention mask at runtime in the forward pass (not great for the performance):
class TransformerModel(nn.Module):
    def __init__(self, num_classes, pad_id, dropout=0.5):
        super(TransformerModel, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-multilingual-cased')
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, num_classes)
        self.relu = nn.ReLU()

        self.pad_id = pad_id
    def forward(self, input_id):
        mask = (input_ids!=self.pad_id).type(input_ids.dtype)

        _, pooled_output = self.bert(input_ids=input_id, attention_mask=mask, return_dict=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        final_layer = self.relu(linear_output)

        return final_layer

model = TransformerModel(num_classes, tokenizer.pad_token_id)


Instruction: Gradient of X is NoneType in second iteration

Input:
I'm trying to make images which will fool model, but I have some problem with this code. In the second iteration I get TypeError: unsupported operand type(s) for -: 'Tensor' and 'NoneType'
Why is the grad NoneType even if it's working for the first time?
X_fooling = X.clone()
X_fooling.requires_grad_()
loss_f = torch.nn.MSELoss()

for i in range(1000):
      score = model(X_fooling)
      y = torch.zeros(1000)
      y[target_y] = 1
      loss = loss_f(score, y)
      print(loss)
      loss.backward()

      if target_y == torch.argmax(score):
        break

      X_fooling = X_fooling - X_fooling.grad

Output:
I could not fool the network with a target size of 1000. But I was able to fool it with a target size of 64. Here is a minimal code snippet that runs without error:
import matplotlib.pyplot as plt
import torch

torch.manual_seed(0)
target_size = 64
model = torch.nn.Linear(10, target_size)
target_y = torch.randint(0, target_size, (1, ))

X = torch.rand(1, 10)

X_fooling = X.clone()
X_fooling.requires_grad_()
loss_f = torch.nn.MSELoss()

history = []
goal_achieved = False

for i in range(10_000):
    score = model(X_fooling)
    y = torch.zeros(1, target_size)
    y[:, target_y] = 1
    loss = loss_f(score, y)
    history.append(loss.detach().cpu().item())
    loss.backward()

    if target_y == torch.argmax(score):
        goal_achieved = True
        break

    X_fooling = (X_fooling - X_fooling.grad).detach()
    X_fooling.requires_grad_()
    model.zero_grad()

plt.title(f"Goal achieved: {goal_achieved}")
plt.plot(history)
plt.show()

Output:

The thing is that you have to detach your input tensor to the graph after your backward pass.


Instruction: What does self(variable) do in Python?

Input:
I'm trying to understand someone else's code in Python and I stumbled across a line I don't quite understand and which I can't find on the internet:
x=self(k)

with k being a torch-array.
I know what self.something does but I haven't seen self(something) before.

Output:
self, for these purposes, is just a variable like any other, and when we call a variable with parentheses, it invokes the __call__ magic method. So
x = self(k)

is effectively a shortcut for
x = self.__call__(k)

Footnote: I say "effectively", because it's really more like
x = type(self).__call__(self, k)

due to the way magic methods work. This difference shouldn't matter unless you're doing funny things with singleton objects, though.


Instruction: Error Running Stable Diffusion from the command line in Windows

Input:
I installed Stable Diffusion v1.4 by following the instructions described in https://www.howtogeek.com/830179/how-to-run-stable-diffusion-on-your-pc-to-generate-ai-images/#autotoc_anchor_2
My machine heavily exceeds the min reqs to run Stable Diffusion:
Windows 11 Pro
11th Gen Intel i7 @ 2.30GHz
Latest NVIDIA GeForce GPU
16GB Memory
1TB SSD
Yet, I get an error when trying to run the test prompt
python scripts/txt2img.py --prompt "a close-up portrait of a cat by pablo picasso, vivid, abstract art, colorful, vibrant" --plms --n_iter 5 --n_samples 1
RuntimeError: CUDA out of memory. Tried to allocate 1024.00 MiB (GPU 0; 8.00 GiB total capacity; 6.13 GiB already allocated; 0 bytes free; 6.73 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
Reading a post by Marco Ramos it seems like it relates to the number of workers in PyTorch
Strange Cuda out of Memory behavior in Pytorch
How do I change the number of workers while running Stable Diffusion? And why is it throwing this error if my machine still has lots of memory? Has anyone encountered this same issue while running Stable Diffusion?

Output:
I had the same issue, it's because you're using a non-optimized version of Stable-Diffusion. You have to download basujindal's branch of it, which allows it use much less ram by sacrificing the precision, this is the branch - https://github.com/basujindal/stable-diffusion
Everything else in that guide stays the same just clone from this version. It allow you to even push past 512x512 default resolution, you can use 756x512 to get rectangular images for example (but the results may vary since it was trained on a 512 square set).
the new prompt becomes python optimizedSD/optimized_txt2img.py --prompt "blue orange" --H 756 --W 512
Also another note: as of a few days ago an even faster and more optimized version was released by neonsecret (https://github.com/basujindal/stable-diffusion), however I'm having issues installing it, so can't really recommend it but you can try it as well and see if it works for you.


Instruction: How to input images in rllib

Input:
last time I saw library rllib: https://docs.ray.io/en/latest/rllib/index.html.
It has amazing features for reinforcement learning, but unfortunately, I couldn't find a way to input images as an observation without flattening them (I basically want to use convolutional neural network). Is there any way to input image observations in models using rllib library?

Output: Rllib is compatible with openai's gym, you can create a custom env https://docs.ray.io/en/latest/rllib/rllib-env.html#configuring-environments and return a Box as an observation space like https://stackoverflow.com/a/69602365/4994352


Instruction: How to make vgg pytorch's ptl size smaller on android?

Input:
import torch
# import joblib
from torch.utils.mobile_optimizer import optimize_for_mobile
from torchvision.models.vgg import vgg16
import torch, torchvision.models

# lb = joblib.load('lb.pkl')
device = torch.device('cuda:0')
#device = torch.device('cpu')#'cuda:0')
torch.backends.cudnn.benchmark = True
model = vgg16().to(device)

# model = torchvision.models.vgg16()
path = 'model-22222.pth'
torch.save(model.state_dict(), path) # nothing else here
model.load_state_dict(torch.load(path))

#model.load_state_dict(torch.load('./model-76-0.7754.pth'))

scripted_module = torch.jit.script(model)
optimized_scripted_module = optimize_for_mobile(scripted_module)
optimized_scripted_module._save_for_lite_interpreter("model-76-0.7754.ptl")

The optimize_for_mobile seems does not make the ptl file smaller, it's about 527M, too large on Android. How to make it smaller?

Output:
You may try using quantization:
https://pytorch.org/docs/stable/quantization.html
Usually it allows to reduce the size of the model 2x-4x times with little or no loss of accuracy.
Example:
import torch
from torch.utils.mobile_optimizer import optimize_for_mobile
from torchvision.models.vgg import vgg16

device = torch.device("cpu")
torch.backends.cudnn.benchmark = True
model = vgg16().to(device)

backend = "qnnpack"
model.qconfig = torch.quantization.get_default_qconfig(backend)
torch.backends.quantized.engine = backend
model_static_quantized = torch.quantization.prepare(model, inplace=False)
model_static_quantized = torch.quantization.convert(
    model_static_quantized, inplace=False
)

scripted_module = torch.jit.script(model_static_quantized)
optimized_scripted_module = optimize_for_mobile(scripted_module)
optimized_scripted_module._save_for_lite_interpreter("model_quantized.ptl")

Apart from it I would try to use the architectures more suitable for mobile phones because VGG is kinda old and has not a great size/accuracy ratio. You can distillate your trained VGG model into smaller one with Knowledge distillation.


Instruction: Best Practices for Distributed Training with PyTorch custom containers (BYOC) in SageMaker

Input: What are the best practices for distributed training with PyTorch custom containers (BYOC) in Amazon Sagemaker? I understand that PyTorch framework supports native distributed training or using the Horovod library for PyTorch.

Output:
The recommended approach on Amazon SageMaker is to use the SageMaker built in Data Parallel and Model Parallel Libraries. When you use the Pytorch Deep Learning container provided by SageMaker, the library is built in and you can follow the below examples to get started with examples.
https://docs.aws.amazon.com/sagemaker/latest/dg/distributed-training-notebook-examples.html
If you are bringing your own container, follow the below link to add SageMaker Distributed training support to your container
https://docs.aws.amazon.com/sagemaker/latest/dg/data-parallel-use-api.html#data-parallel-bring-your-own-container
Apart from this SageMaker also natively supports Pytorch DDP within the native Deep Learning Container used in Pytorch Estimator.
https://sagemaker.readthedocs.io/en/stable/frameworks/pytorch/sagemaker.pytorch.html


Instruction: PyTorch convert function for op 'pad' not implemented

Input:
When I trying to convert model.ckpt to core ML model use coremltools I got this error:

  File "/Users/peterpan/miniforge3/lib/python3.10/site-packages/coremltools/converters/mil/frontend/torch/ops.py", line 86, in convert_nodes
    raise RuntimeError(
RuntimeError: PyTorch convert function for op 'pad' not implemented.

Here is converting code:
   model: torch.nn.Module = make_training_model(train_config)
    state = torch.load(path, map_location=map_location)
    model.load_state_dict(state['state_dict'], strict=strict)
    model.on_load_checkpoint(state)
    model.eval()

    jit_model_wrapper = JITWrapper(model)
    image = torch.rand(1, 3, 120, 120)
    mask = torch.rand(1, 1, 120, 120)
    output = jit_model_wrapper(image, mask)

    device = torch.device("cpu")

    image = image.to(device)
    mask = mask.to(device)
    traced_model = torch.jit.trace(jit_model_wrapper, (image, mask), strict=False).to(device)
    model1 = ct.convert(
        traced_model,
        source='pytorch',
        inputs=[ct.ImageType(name='image',shape=image.shape), ct.ImageType(name='mask',shape=mask.shape)]
    )
    model1.save("newmodel.mlmodel")

I'm newbie in python. What's wrong with my code?

Output: Upgrading to coremltools==6.0 fixed this problem for me


Instruction: Find euclidean distance between a tensor and each row tensor of a matrix efficiently in PyTorch

Input:
I have a tensor A of size torch.Size([3]) and another tensor B of size torch.Size([4,3]).
I want to find the distance between A and each of the 4 rows of B.
I'm new to Torch and I reckon a for loop for each of the rows wouldn't be efficient. I have looked into torch.linalg.norm and torch.cdist but I'm not sure if they solve my problem, unless I'm missing something.

Output:
You look for:
torch.norm(A[None, :] - B, p=2, dim=1)

A[None, :] resize the tensor to shape (1, 3)
A[None, :] - B will copy 4 times the tensor A to match the size of B ("broadcast") and make the substraction
torch.norm(..., p=2, dim=1) computes the euclidian norme for each column.

Output shape: (4,)


Instruction: AttributeError: 'str' object has no attribute 'cuda'

Input:
I am trying to move my data to GPU by doing this
batch["img"] = [img.cuda() for img in batch["img"]]
batch["label"] = [label.cuda() for label in batch["label"]]

However, i get this error for the labels for OCR
AttributeError: 'str' object has no attribute 'cuda'

I also tried .to('cuda') and similar error.
More details are as follows. This is the pytorch Dataset class
class SynthDataset(Dataset):
    def __init__(self, opt):
        super(SynthDataset, self).__init__()
        self.path = os.path.join(opt['path'], opt['imgdir'])
        self.images = os.listdir(self.path)
        self.nSamples = len(self.images)
        f = lambda x: os.path.join(self.path, x)
        self.imagepaths = list(map(f, self.images))
        transform_list =  [transforms.Grayscale(1),
                            transforms.ToTensor(),
                            transforms.Normalize((0.5,), (0.5,))]
        self.transform = transforms.Compose(transform_list)
        self.collate_fn = SynthCollator()
    def __len__(self):
        return self.nSamples

    def __getitem__(self, index):
        assert index <= len(self), 'index range error'
        imagepath = self.imagepaths[index]

        imagefile = os.path.basename(imagepath)
        img = Image.open(imagepath)
        if self.transform is not None:
            img = self.transform(img)
        item = {'img': img, 'idx':index}
        item['label'] = imagefile.split('_')[0]
        return item

As you can see the generator outputs a dictionary with image and labels where the labels is the text contained in the image

Output:
Yo !
If I get your code right, your are building your custom dataset to output an image and its label. Your label comes from the first part of your image filename, so it is a string, as stated in the error message. Your object needs to be a Torch tensor to be moved on the GPU with .cuda().
If you want to keep your code as is, you need to transform your label into a vector form. I suspect your labels to be stuff like "cat", "dog", etc The usual way to transform labels into numerical forms are one-hot-encoding, or simply map each label to an integer. There is abundant resources about it on the web. Then you can turn your label into a Tensor object and move it to the GPU.
However, I would highly recommand to change the way you build your tensor dataset. You are calling .cuda() on every image tensor in your list. When forwarding a neural network, you batch your data into a tensor (for instance stacking your images into a new dimension), then call .cuda() on the whole batch tensor. Same thing for the labels. Check the Pytorch documentation as there is probably the exact example you are looking for (a custom dataset with an image and its label).
Good luck !


Instruction: Why we use validation set (not train or test set) in early stopping function ( DL / CNN )?

Input:
This is my first attempt to CNN in Pytorch. I have gone by few tutorials, but still need some clarification.
I have theoretical question, I don't understand why in early stopping function we base on validation set, not train or test set?
Has it something common with metrics we got from validation set?

Output:
The number of training epochs is one of the training hyper-parameters. Therefore, you MUST NOT use the test data to determine the value of this hyper-parameter.
Additionally, you cannot use the training set itself to determine the value of early stopping. Therefore, you need to use the validation set for determining this value.


Instruction: How to set backend to ‘gloo’ on windows in Pytorch

Input:
I am trying to use two gpus on my windows machine, but I keep getting

raise RuntimeError("Distributed package doesn't have NCCL " "built
in") RuntimeError: Distributed package doesn't have NCCL built in

I am still new to pytorch and couldnt really find a way of setting the backend to ‘gloo’. Any way to set backend= 'gloo' to run two gpus on windows.

Output:
from torch import distributed as dist

Then in your init of the training logic:
dist.init_process_group("gloo", rank=rank, world_size=world_size)

Update:
You should use python multiprocess like this:
class Trainer:
    def __init__(self, rank, world_size):
        self.rank = rank
        self.world_size = world_size
        self.log('Initializing distributed')
        os.environ['MASTER_ADDR'] = self.args.distributed_addr
        os.environ['MASTER_PORT'] = self.args.distributed_port
        dist.init_process_group("gloo", rank=rank, world_size=world_size)

if __name__ == '__main__':
    world_size = torch.cuda.device_count()
    mp.spawn(
        Trainer,
        nprocs=world_size,
        args=(world_size,),
        join=True)


Instruction: Pytorch BERT input gradient

Input:
I am trying to get the input gradients from a BERT model in pytorch. How can I do that?
Suppose, y' = BertModel(x). I am trying to find $d(loss(y,y'))/dx$

Output:
One of the problems with Bert models is that your input mostly contains token IDs rather than token embeddings, which makes getting gradient difficult since the relation between token ID and token embeddings is discontinued.
To solve this issue, you can work with token embeddings.
# get your batch data: token_id, mask and labels
token_ids, mask, labels = batch

# get your token embeddings
token_embeds=BertModel.bert.get_input_embeddings().weight[token_ids].clone()
# track gradient of token embeddings
token_embeds.requires_grad=True

# get model output that contains loss value
outs = BertModel(inputs_embeds=inputs_embeds,labels=labels)
loss=outs.loss

After getting loss value, you can use torch.autograd.grad in this answer or backward function
loss.backward()
grad=token_embeds.grad


Instruction: What is the unit of time in the following code?

Input:
The following code shows inference time of the network. However, I am not sure what the unit of time is. Read through the documentation here, but still confused
####This code is for measuring performance such as inference time, FLOPs, Parameters etc...
    dummy_input = torch.randn(1, 3, 256, 256).cuda()

    macs, params = profile(model, inputs=(dummy_input,), verbose=0)
    macs, params = clever_format([macs, params], "%.3f")
    name = "SegDepthWithTwoDecoders"
    print("<" * 50, name)
    print("Flops:", macs)
    print("Parameters:", params)

    starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(
        enable_timing=True
    )

    repetitions = 300
    timings = np.zeros((repetitions, 1))

    for _ in range(10):
        _ = model(dummy_input)

    # MEASURE PERFORMANCE
    with torch.no_grad():
        for rep in range(repetitions):
            starter.record()
            _ = model(dummy_input)
            ender.record()
            # WAIT FOR GPU SYNC
            torch.cuda.synchronize()
            curr_time = starter.elapsed_time(ender)
            timings[rep] = curr_time
    print("time :", np.average(timings))
    ###This is where the code ends for measuring the performance

This is what the output looks like
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< SegDepthWithTwoDecoders
Flops: 1.924G
Parameters: 3.986M
time : 12.67153577486674

Output:
From the documentation of Event.elapsed_time()
https://pytorch.org/docs/stable/generated/torch.cuda.Event.html

Returns the time elapsed in milliseconds after the event was recorded
and before the end_event was recorded.


Instruction: Spyder crashing when importing torch

Input:
I am using a MacBook Pro (MacOS: Monterey) and I'm using Spyder downloaded as the app for MacOS via this page: https://github.com/spyder-ide/spyder/releases. So it is from a standalone installer and I have installed conda via miniconda3.
Everything works fine until I'm trying to install Pytorch. I have installed the package in a virtual environment with the following code snippet: conda install pytorch torchvision -c pytorch.
The installation is successful but when I write import torch, I get the following error message and the kernel restarts:
/Applications/Spyder.app/Contents/Resources/lib/python3.9/spyder/plugins/ipythonconsole/scripts/conda-activate.sh: line 18: 98840 Abort trap: 6

$CONDA_ENV_PYTHON -m spyder_kernels.console -f $SPYDER_KERNEL_SPEC

Fatal Python error: Aborted

Main thread:
Current thread 0x0000000112f1f600 (most recent call first):
  File "<frozen importlib._bootstrap>", line 241 in _call_with_frames_removed
  File "<frozen importlib._bootstrap_external>", line 1176 in create_module
  File "<frozen importlib._bootstrap>", line 571 in module_from_spec
  File "<frozen importlib._bootstrap>", line 674 in _load_unlocked
  File "<frozen importlib._bootstrap>", line 1006 in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 1027 in _find_and_load
  File "/Users/andreasaspe/opt/miniconda3/envs/spyder-env/lib/python3.10/site-packages/torch/__init__.py", line 202 in <module>
  File "<frozen importlib._bootstrap>", line 241 in _call_with_frames_removed
  File "<frozen importlib._bootstrap_external>", line 883 in exec_module
  File "<frozen importlib._bootstrap>", line 688 in _load_unlocked
  File "<frozen importlib._bootstrap>", line 1006 in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 1027 in _find_and_load
  File "/var/folders/fk/q49x7w9j6t53t4bvkbj_nkdm0000gp/T/ipykernel_98840/4265195184.py", line 1 in <module>

Restarting kernel...

Note: If I activate the virtual environment in the terminal and run a python-script here, then pytorch works absolutely fine. And I have tried a few times that the python-script also suddenly starts running with no problems in the Spyder console. But when I close down Spyder and starts it again, then the issue starts all over and I cannot find a pattern for why it some times works. I don't know if I'm doing anything wrong regarding my virtual environment? I have changed my python interpreter inside of Spyder to be the one in my virtual environment.. Since it works in the terminal and not in the spyder-console I also suspect that it can be something with my spyder-kernel (as the error also suggest). But I can't really figure out how to fix the spyder-kernel.
I have tried to uninstall and install again, installing with pip instead of conda but nothing works. I have searched the internet and for other people it helped to update Spyder to the newest version and making sure that Pytorch is of the newest version as well. It seems like I have the newest editions of everything, though.
Information about Spyder (as standalone installer):
Spyder IDE: 5.3.3
Python 3.9.5 64-bit | Qt 5.15.2 | PyQt5 5.15.7 | Darwin 21.5.0
Information about Pytorch package:
Version 1.12.1

Output:
Question:
Spyder crashing when importing torch on MacOS M2
Answer:
At the new update Anaconda @ MacOS Monterey it works with downgrading pytorch==1.7.1 for spyder==5.3.3.
$ conda install pytorch==1.7.1


Instruction: RuntimeError: CUDA out of memory. How setting max_split_size_mb?

Input:
I found this problem running a neural network on Colab Pro+ (with the high RAM option).
RuntimeError: CUDA out of memory. Tried to allocate 8.00 GiB (GPU 0; 15.90 GiB total capacity; 12.04 GiB already allocated; 2.72 GiB free; 12.27 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
I have already decreased the batch to 2. I upload the data using the h5py.
At this point, I assume the only thing I can try is setting the max_split_size_mb.
I could not find anything about how I can implement the max_split_size_mb. The Pytorch documentation (https://pytorch.org/docs/stable/notes/cuda.html) was not clear to me.
Anyone can support me?
Thank you.

Output:
The max_split_size_mb configuration value can be set as an environment variable.
The exact syntax is documented at https://pytorch.org/docs/stable/notes/cuda.html#memory-management, but in short:

The behavior of caching allocator can be controlled via environment variable PYTORCH_CUDA_ALLOC_CONF. The format is PYTORCH_CUDA_ALLOC_CONF=<option>:<value>,<option2>:<value2>...
Available options:

max_split_size_mb prevents the allocator from splitting blocks larger than this size (in MB). This can help prevent fragmentation and may allow some borderline workloads to complete without running out of memory. Performance cost can range from ‘zero’ to ‘substatial’ depending on allocation patterns. Default value is unlimited, i.e. all blocks can be split. The memory_stats() and memory_summary() methods are useful for tuning. This option should be used as a last resort for a workload that is aborting due to ‘out of memory’ and showing a large amount of inactive split blocks.

...

So, you should be able to set an environment variable in a manner similar to the following:
Windows: set 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'
Linux: export 'PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512'
This will depend on what OS you're using - in your case, for Google Colab, you might find Setting environment variables in Google Colab helpful.


Instruction: Approximating an exponential fit with a simple neural network

Input:
I've been trying to train a network to solve exponential fits of the form s(t) = s0 * e^(-t/decay_constant). As input, the net takes s and t, and as output it should return s0 and the decay_constant.
This seems like a sufficiently simple problem that I would expect a net would be able to satisfactorely approximate.
However, I cannot get it to work. The loss does go down, and the results don't look completely random, but they are definitely worse than what I simple log-linear fit could achieve.
My setup was (takes about 20s on CPU)

A dense net with ReLU activations
Loss based on log-linear LSQ
Train on random exponential decay examples

import torch

net = torch.nn.Sequential(
    torch.nn.Linear(10, 64), torch.nn.ReLU(),
    torch.nn.Linear(64, 64), torch.nn.ReLU(),
    torch.nn.Linear(64, 32), torch.nn.ReLU(),
    torch.nn.Linear(32, 2), torch.nn.ReLU(),  # Want outputs to always be positive.
)

loss = torch.nn.MSELoss()
optimizer = torch.optim.Adam(net.parameters(), lr=0.005)

def signal_model(x, s0, decay):
    return s0 * torch.exp(-x / decay)

# Generate some datapoints
batch_size = 4096

for episode in range(1000):
    # Generate random time, decay_constant s0.
    t = t = (torch.arange(5, 55, 10.) + (torch.rand(batch_size, 5) * 10)).T
    decay_constant = torch.rand(batch_size) * 70 + 10
    s0 = (torch.rand(batch_size) * 2 - 1) * 50 + 100

    # Generate random input data
    y = signal_model(t, s0, decay_constant)
    data = torch.vstack((t, y)).T

    # Make a prediction and calculate loss
    coefficients = net(data)
    l = loss(-torch.log(signal_model(t, *coefficients.T) + 1e-20), -torch.log(y + 1e-20))

    optimizer.zero_grad(); l.backward(); optimizer.step()

To visualize results
import matplotlib.pyplot as plt

t_ = torch.tensor([5, 15, 25, 35, 45], dtype=torch.float32)
plotgrid = torch.arange(0, 50, 0.1)
s = signal_model(t_, 20, 30)

fig, ax = plt.subplots()
ax.plot(plotgrid, signal_model(plotgrid, 20, 30).detach().numpy())
ax.scatter(t_, s.detach().numpy(), label="True")
ax.scatter(
    t_,
    signal_model(t_, *net(torch.hstack((t_, s)).T)).detach().numpy(),
    label="Predicted",
)
ax.legend()

I was wondering whether anyone has insights on why this doesn't work. Is there some fundamental limitation to approximating simple functions? Or is there something that jumps to the eye as inherently wrong? Would love any insights.
What didn't work:

tweaking learning rate
tweaking number of layers / units per layer
tweaking batch_size
changing loss to MSE on coefficients or on the signal directly without log
regularizing with L1loss
using always the same t as input

Somewhat related question: https://stats.stackexchange.com/questions/379884/why-cant-a-single-relu-learn-a-relu

Output:
s0 and decay_constant are not fixed parameter values in your generated data. There is not a single true parameter vector for the model to converge to. You might think that giving it a variety of different examples of exponential outputs from different parameters means it is being trained to predict proper exponential fit coefficients from any given dataset, but that's wrong. Rather it is just being trained to reproduce a specific distribution of exponential fit coefficients (what looks like normally distributed data with mean 10 and variance of 490 for decay_constant and mean 100 with variance of 2500 for s0). As a thought experiment, what if we fed in an example data set with parameters way way way far out in the tails of those distributions? The model couldn't be adequate at guessing that relationship given that that part of the (s0, decay_constant) space would be essentially missing entirely from the randomized training set. Think of the space of all possible exponential curves ...
The key thing you are missing is that you can't make a single model that predicts the coefficients for any possible dataset. Rather, you can train the model on any dataset to predict the coefficients for that dataset. If the training data is generated from a sample where those coefficients are fixed, then the model's loss optimization will cause the predicted coefficients to converge to the true coefficients for that dataset. You'd essentially be hijacking a NN optimizer to do what a more common optimizer would do with just the deterministic equation solving you'd normally do.
But if you hand a dataset with no fixed parameters, the outputs can at best represent the distribution of parameters according to however you defined it when making the synthetic training data. Because it's not possible to make a training set that adequately includes every possible exponential curve there can be, the model will perform poorly in general. It may perform OK for curve fitting where the parameters are close to the synthetic training distribution you made, but this will depend a lot of the variance and the quality of convergence.
I made a similar type of model for linear regression coefficient predictions. Note: not fitting a linear model, rather jumping straight to predicting the coefficients of a linear model from a dataset generated by a fixed, true, set of coefficients. It is very similar in spirit to what you're trying to do here, but note how it must be trained on the one single dataset you want the coefficients for. It cannot predict generic coefficients across all possible linear relationships. To get meaningful outputs, you don't just make single predictions - rather you would do something like average the predictions across a test set held out from the same data generated by the fixed, true parameters.

https://github.com/spearsem/nn4params


Instruction: Is Cuda Toolkit release 11.7 compatible with pytorch version 11.6?

Input: I have installed recent version of cuda toolkit that is 11.7 but now while downloading I see pytorch 11.6 is there, are they two compatible?

Output:
There is a table with CUDA compatibility:
https://pytorch.org/get-started/locally/
At this moment the latest supported CUDA version is 11.6.


Instruction: Why the training error is greater than the test error?

Input:
I have trained a model in mode .train() (nn.Models of Pytorch)
I have saved the BCELoss (in mode .train()) and the accuracies on ds test and on ds train (but in mode .eval() ).
So, the results were:
(for example in epoch 153)
error classification (train, test): 0.5819954128440368 , 0.37209302325581395
and the loss was:
0.0032386823306408906
How was it possibile?
Was I wrong to switch the two modes? Is this the problem?

Output:
Test error is not always greater than train error, but it seems that your loss function or model structure has some problem considering that you trained your model for 153 epoches.
Why don't you design a new layer structure, referring lots of publication ? In my experience your problem usually occurs when your model is not the optimized model for your dataset. It is not solved by simply enhancing the depth of model.


Instruction: How to turn a numpy array (mic/loopback input) into a torchaudio waveform for a PyTorch classifier

Input:
I am currently working on training a classifier with PyTorch and torchaudio. For this purpose I followed the following tutorial: https://towardsdatascience.com/audio-deep-learning-made-simple-sound-classification-step-by-step-cebc936bbe5
This all works like a charm and my classifier is now able to successfully classify .wav files.
However I would like to turn this into a real-time classifier, that is able to also classify recordings from a microphone/loopback input.
For this I would hope to not have to save a recording into a .wav file to load it again but instead directly feed the classifier with an in memory recording.
The tutorial uses the .load function of torchaudio to load a .wav file and return a waveform and sample rate as follows:
sig, sr = torchaudio.load(audio_file)

Now loopback is pretty much required and since pyaudio does apparently not support loopback devices yet (except for a fork that is very likely to be outdated) I stumbled across soundcard:
https://soundcard.readthedocs.io/en/latest/
I found this code to yield a recording of my speaker loopback:
speakers = sc.all_speakers()
# get the current default speaker on your system:
default_speaker = sc.default_speaker()

# get a list of all microphones:v
mics = sc.all_microphones(include_loopback=True)

# get the current default microphone on your system:
default_mic = mics[0]

with default_mic.recorder(samplerate=148000) as mic, \
            default_speaker.player(samplerate=148000) as sp:
    print("Recording...")
    data = mic.record(numframes=1000000)
    print("Done...Stop your sound so you can hear playback")
    time.sleep(5)
    sp.play(data)

However now of course I don't want to play that audio with the .play function but instead pass it onto to torchaudio/the classifier.
Since I am new to the world of audio processing I have no idea how to get this data into a suitable format similar to the one returned by torchaudio.
According to the docs of soundcard the data has the following format:
The data will be returned as a frames × channels float32 numpy array
As a last resort maybe saving it into an in memory .wav file and then reading it with torchaudio is possible?
Any help is appreciated. Thank you in advance!

Output:
According to the doc, you will get a numpyarray of shape  frames × channels. For a stereo microphone, this will be (N,2), for mono microphone (N,1).
This is pretty much what the torch load function outputs: sig is a raw signal, and sr the sampling rate. You have specified your sample rate yourself
to your mic (so sr = 148000), and you just need to convert your numpy raw signal to a torch tensor with:
sig_mic = torch.tensor(data)

Just check that the dimensions are similar, it might be something like (2,N) with torchaudio.load(), in such case, just reshape the tensor:
sig_mic = torch.tensor(data).reshape((2, -1))


Instruction: How to vectorize a torch function?

Input:
When using numpy I can use np.vectorize to vectorize a function that contains if statements in order for the function to accept array arguments. How can I do the same with torch in order for a function to accept tensor arguments?
For example, the final print statement in the code below will fail. How can I make this work?
import numpy as np
import torch as tc

def numpy_func(x):
    return x if x > 0. else 0.
numpy_func = np.vectorize(numpy_func)

print('numpy function (scalar):', numpy_func(-1.))
print('numpy function (array):', numpy_func(np.array([-1., 0., 1.])))

def torch_func(x):
    return x if x > 0. else 0.

print('torch function (scalar):', torch_func(-1.))
print('torch function (tensor):', torch_func(tc.tensor([-1., 0., 1.])))

Output:
You can use .apply_() for CPU tensors. For CUDA ones, the task is problematic: if statements aren't easy to SIMDify.
You may apply the same workaround for functorch.vmap as video drivers used to do for shaders: evaluate both branches of the condition and stick to arithmetics.
Otherwise, just use a for loop: that's what np.vectorize() mostly does anyway.
def torch_vectorize(f, inplace=False):
    def wrapper(tensor):
        out = tensor if inplace else tensor.clone()
        view = out.flatten()
        for i, x in enumerate(view):
            view[i] = f(x)
        return out
    return wrapper


Instruction: ValueError: too many values to unpack (expected 2) while trying to load yolov5 model

Input:
I am trying to load a trained yolov5 model on a custom dataset using this:
# Model
model = torch.hub.load('/home/yolov5/runs/train/yolo_sign_det2/weights', 'best')  # or yolov5n - yolov5x6, custom

but I am running into this error:
ValueError                                Traceback (most recent call last)
<ipython-input-3-c832ab8c1eab> in <module>
      2
      3 # Model
----> 4 model = torch.hub.load('/home/yolov5/runs/train/yolo_sign_det2/weights', 'best')  # or yolov5n - yolov5x6, custom
      5
      6 # Images

~/.conda/envs/yolo/lib/python3.6/site-packages/torch/hub.py in load(repo_or_dir, model, source, force_reload, verbose, skip_validation, *args, **kwargs)
    395
    396     if source == 'github':
--> 397         repo_or_dir = _get_cache_or_reload(repo_or_dir, force_reload, verbose, skip_validation)
    398
    399     model = _load_local(repo_or_dir, model, *args, **kwargs)

~/.conda/envs/yolo/lib/python3.6/site-packages/torch/hub.py in _get_cache_or_reload(github, force_reload, verbose, skip_validation)
    163         os.makedirs(hub_dir)
    164     # Parse github repo information
--> 165     repo_owner, repo_name, branch = _parse_repo_info(github)
    166     # Github allows branch name with slash '/',
    167     # this causes confusion with path on both Linux and Windows.

~/.conda/envs/yolo/lib/python3.6/site-packages/torch/hub.py in _parse_repo_info(github)
    110     else:
    111         repo_info, branch = github, None
--> 112     repo_owner, repo_name = repo_info.split('/')
    113
    114     if branch is None:

ValueError: too many values to unpack (expected 2)

Can anybody please tell what I'm doing wrong?

Output:
From the Pytorch documentation website, it seems that the source option is set to github by default so your line of code:
model = torch.hub.load('/home/yolov5/runs/train/yolo_sign_det2/weights', 'best')  # or yolov5n - yolov5x6, custom

actually means:
model = torch.hub.load('/home/yolov5/runs/train/yolo_sign_det2/weights', 'best', source='github')  # or yolov5n - yolov5x6, custom

Thus, you need to set source=local as explained in the linked documentation, by keeping source='github' it means your programme is trying to find a GitHub repository (as can be seen in your error messages).
The fix is to use this line of code instead with the source set to local:
torch.hub.load('/home/yolov5/runs/train/yolo_sign_det2/weights', 'best', source='local')


Instruction: How to convert pytorch model to being on GPU?

Input:
I want to run pytorch on a GPU.
I have this code:
import torch
import torch.nn as nn
device = torch.device("cuda:0")

n_input, n_hidden, n_out, batch_size, learning_rate = 10, 15, 1, 100, 0.01

data_x = torch.randn(batch_size, n_input)
data_y = (torch.rand(size=(batch_size, 1)) < 0.5).float()

print(data_x.size())
print(data_y.size())

model = nn.Sequential(nn.Linear(n_input, n_hidden),
                      nn.ReLU(),
                      nn.Linear(n_hidden, n_out),
                      nn.Sigmoid())

#model.to(device)
print(next(model.parameters()).is_cuda)

loss_function = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

losses = []
for epoch in range(5000):
    pred_y = model(data_x)
    loss = loss_function(pred_y, data_y)
    losses.append(loss.item())

    model.zero_grad()
    loss.backward()
    optimizer.step()

print(torch.cuda.get_device_name())
print(torch.__version__)
print(torch.version.cuda)

import matplotlib.pyplot as plt
plt.plot(losses)
plt.ylabel('loss')
plt.xlabel('epoch')
plt.title("Learning rate %f"%(learning_rate))
plt.show()

When I run it, the output is:
torch.Size([100, 10])
torch.Size([100, 1])
False
Quadro P2000
1.12.1
11.3

when I uncomment the line model.to(device) and rerun it, I get:
Traceback (most recent call last):
  File "basic_pytorch_with_gpu.py", line 26, in <module>
    pred_y = model(data_x)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/container.py", line 139, in forward
    input = module(input)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 114, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking argument for argument mat1 in method wrapper_addmm)

I can see other questions like this (e.g. here) but can't work out how I'm meant to convert to GPU.
For example, I change the data_x and data_y lines to:
data_x = torch.randn(batch_size, n_input).cuda()
data_y = (torch.rand(size=(batch_size, 1)) < 0.5).float().cuda()

But I get the same error - could someone explain how to run this code on a GPU?

Output:
As suggested in the comments, you need to transfer both your model and your data to the same device.  Below should work:
import torch
import torch.nn as nn
device = torch.device("cuda:0")

n_input, n_hidden, n_out, batch_size, learning_rate = 10, 15, 1, 100, 0.01

data_x = torch.randn(batch_size, n_input)
data_y = (torch.rand(size=(batch_size, 1)) < 0.5).float()

print(data_x.size())
print(data_y.size())

model = nn.Sequential(nn.Linear(n_input, n_hidden),
                      nn.ReLU(),
                      nn.Linear(n_hidden, n_out),
                      nn.Sigmoid())

# Transfer to device
model = model.to(device)
data_x = data_x.to(device)
data_y = data_y.to(device)

print(next(model.parameters()).is_cuda)

loss_function = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

losses = []
for epoch in range(5000):
    pred_y = model(data_x)
    loss = loss_function(pred_y, data_y)
    losses.append(loss.item())

    model.zero_grad()
    loss.backward()
    optimizer.step()

print(torch.cuda.get_device_name())
print(torch.__version__)
print(torch.version.cuda)

import matplotlib.pyplot as plt
plt.plot(losses)
plt.ylabel('loss')
plt.xlabel('epoch')
plt.title("Learning rate %f"%(learning_rate))
plt.show()


Instruction: python methods in comparison

Input:
Rock, Paper, Scissors variable bot has default value
Variable Alex, has values that are passed to main.py
When I call method comparison I get an error
Method comparisons
from secrets import choice
from variants import Variants
Player.py
class Player:
    name = '',
    choice = ''

    def __init__(self, choise = 'ROCK', name = 'bot'):
        self.name = name
        self.choice = choice

    def whoWins(self, bot, alex):
        if bot.choice > alex.choice:
            print('bot, winner')
        if bot.choice < alex.choice:
            print('Alex, winner')
        if bot.choice == alex.choice:
            print('draw')

main.py
from variants import Variants
from player import Player

bot = Player()
alex = Player(Variants.ROCK, "Alex")
print(bot.whoWins(bot, alex))

variants.py
from enum import Enum

class Variants(Enum):
    ROCK = 1,
    PAPER = 2,
    SCISSORS = 3

Output:
Answer:
I had to modify the comparison of methods, I did it like this:
also main.po remained the same:
> from variants import Variants from player import Player
>
> bot = Player() alex = Player(Variants.ROCK, "Alex")
> print(bot.whoWins(bot, alex))

variants.py remained the same
from enum import Enum

class Variants(Enum):
    ROCK = 1
    PAPER = 2
    SCISSORS = 3

Player.py
from secrets import choice
from variants import Variants

class Player:
    name = '',
    choice = ''

    def __init__(self, choice = Variants.ROCK,  name = 'bot'):
        self.name = name
        self.choice = choice

    def whoWins(self, bot, alex):
        if (bot.choice == Variants.ROCK and alex.choice == Variants.PAPER):
            print('Alex, win!')
        if (bot.choice == Variants.PAPER and alex.choice == Variants.ROCK):
            print('Alex, win!')
        if (bot.choice == Variants.SCISSORS and alex.choice == Variants.SCISSORS):
            print('draw')
        if (bot.choice == Variants.ROCK and alex.choice == Variants.ROCK):
            print('draw!')
        if (bot.choice == Variants.ROCK and alex.choice == Variants.SCISSORS):
            print('Bot, win')
        if (bot.choice == Variants.SCISSORS and alex.choice == Variants.PAPER):
            print('Bot, win')
        if (bot.choice == Variants.SCISSORS and alex.choice == Variants.ROCK):
            print('Alex, win!')
        if (bot.choice == Variants.PAPER and alex.choice == Variants.SCISSORS):
            print('Alex, win!')
        elif (bot.choice == Variants.SCISSORS and alex.choice == Variants.SCISSORS):
            print('draw!')

if bot.choice > alex.choice:    TypeError: '>' not supported between instances of 'method' and 'method' - The error is gone
All, is ok!
PS C:\Users\user\2> python.exe main.py
Alex, win!
PS PS C:\Users\user\2>


Instruction: Pytorch: Why does evaluating a string (of an optimizer) in a function break the function?

Input:
I have a pytorch lightning class that looks like this:
import torch.optim as optim

class GraphLevelGNN(pl.LightningModule):

def __init__(self,**model_kwargs):
    super().__init__()

    self.save_hyperparameters()
    self.model = GraphGNNModel(**model_kwargs)
    self.loss_module = nn.BCEWithLogitsLoss()
    self.optimizer = eval('optim.SGD(self.parameters(),lr=0.1)')

def forward(self, data, mode="train"):
    x, edge_index, batch_idx = data.x, data.edge_index, data.batch
    x = self.model(x, edge_index, batch_idx)
    x = x.squeeze(dim=-1)

    if self.hparams.c_out == 1:
        preds = (x > 0).float()
        data.y = data.y.float()
    else:
        preds = x.argmax(dim=-1)

    loss = self.loss_module(x, data.y)
    acc = (preds == data.y).sum().float() / preds.shape[0]

    return loss, acc,preds

def configure_optimizers(self):
    optimizer = self.optimizer
    return optimizer

def training_step(self, batch, batch_idx):
    loss, acc _ = self.forward(batch, mode="train")
    self.log('train_loss', loss,on_epoch=True,logger=True,batch_size=64)
    self.log('train_acc', acc,on_epoch=True,logger=True,batch_size=64)

def validation_step(self, batch, batch_idx):
    loss, acc,_ = self.forward(batch, mode="val")
    self.log('val_acc', acc,on_epoch=True,logger=True,batch_size=64)
    self.log('val_loss', loss,on_epoch=True,logger=True,batch_size=64)

def test_step(self, batch, batch_idx):
    loss,acc, preds = self.forward(batch, mode="test")
    self.log('test_acc', acc,on_epoch=True,logger=True,batch_size=64)

I eventually want to put the optimizer into a ray tune object, so I want it to not be hard coded in this function.
Why is it that when I have:
 self.optimizer = optim.SGD(self.parameters(),lr=0.1)

in the __init__ part, the script works, but when I change to eval('optim.SGD(self.parameters(),lr=0.1)'), then the function breaks with the error:
  File "script.py", line 560, in __init__
    self.optimizer = eval('optim.SGD(self.parameters(),lr=0.1)')
  File "<string>", line 1, in <module>
NameError: name 'optim' is not defined

I also tried changing optim to torch.optim but it produces the same error.
Should the eval not change the string 'optim.SGD(self.parameters(),lr=0.1)' to optim.SGD(self.parameters(),lr=0.1)

Output:
Even though using eval is not the correct approach to your problem, let me just explain why you are facing this error.
The python function eval does not import any modules or functions in the script you are running by default.
For example, you can take eval as the python interpreter, when you just open the interpreter, and do not import any modules or write any functions, it will give you an error when you try and use them. Like if you use time.sleep() function without importing time, it gives you an error.
To overcome this, it allows you to assign variables in the arguments of the eval function.
eval(expression[, globals[, locals]])

You can read about it here -
https://docs.python.org/3/library/functions.html#eval
https://www.programiz.com/python-programming/methods/built-in/eval


Instruction: Performance gap between `batch_size==32` and `batch_size==8, gradient_accumulation==4`

Input:
I tried to use gradient accumulation in my project. To my understanding, the gradient accumulation is the same as increasing the batch size by x times. I tried batch_size==32 and batch_size==8, gradient_accumulation==4 in my project, however the result varies even when I disabled shuffle in dataloader. The batch_size==8, accumulation==4 variant's result is significantly poorer.
I wonder why?
Here is my snippet:
loss = model(x)
epoch_loss += float(loss)

loss.backward()

# step starts from 1
if (step % accumulate_step == 0) or (step == len(dataloader)):

    if clip_grad_norm > 0:
        nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)

    optimizer.step()
    if scheduler:
        scheduler.step()

    optimizer.zero_grad()

Output:
Assuming your loss is mean-reduced, then you need to scale the loss by 1/accumulate_step
The default behavior of most loss functions is to return the average loss value across each batch element. This is referred to as mean-reduction, and has the property that batch size does not affect the magnitude of the loss (or the magnitude of the gradients of loss). However, when implementing gradient accumulation, each time you call backward you are adding gradients to the existing gradients. Therefore, if you call backward four times on quarter-sized batches, you are actually producing gradients that are four-times larger than if you had called backward once on a full-sized batch. To account for this behavior you need to divide the gradients by accumulate_step, which can be accomplished by scaling the loss by 1/accumulate_step before back-propagation.
loss = model(x) / accumulate_step

loss.backward()

# step starts from 1
if (step % accumulate_step == 0) or (step == len(dataloader)):

    if clip_grad_norm > 0:
        nn.utils.clip_grad_norm_(model.parameters(), max_norm=clip_grad_norm)

    optimizer.step()
    if scheduler:
        scheduler.step()

    optimizer.zero_grad()


Instruction: A simple question about torch.einsum function

Input:
A = torch.randn(5,5)
B = torch.einsum("ii->i",A)
C = torch.einsum("ii",A)

Just I exhibit above,I know the result about B that means getting the diagonal elements.
print("before:",A)
print("after:",B)
print('Why:',C)

results
before: tensor([[-0.2339,  0.2501, -1.1814,  1.4392, -0.5461],
        [ 1.4908,  0.0626, -0.6849, -1.3106,  0.1257],
        [ 3.3362, -1.7438,  0.3027,  0.4346,  0.6830],
        [-0.6183,  0.5965,  1.2653,  1.0319, -0.0670],
        [-0.5531, -0.4245, -2.4869,  1.2972,  0.6732]])
after: tensor([-0.2339,  0.0626,  0.3027,  1.0319,  0.6732])
Why: tensor(1.8366)

So,Why tensor C is 1.8366?

Output:
Executing torch.einsum("ii", A) is equivalent to torch.einsum("ii->", A), this means the output has no index. You can interpret the output as a rank zero tensor.
So this corresponds to computing the sum of the diagonal elements.


Instruction: How can I expand a tensor in Libtorch? (The C++ version of PyTorch)

Input:
How can I use LibTorch to expand a tensor of the shape 42, 358 into a shape of 10, 42, 358?
I know how to do this in PyTorch, (AKA Torch).
torch.ones(42, 358).expand(10, -1, -1).shape
returns
torch.Size([10, 42, 358])
In LibTorch I have a tensor of the same size I am trying to "expand" in the same way.
    auto expanded_state_batch = state_batch.expand(10, -1, -1);

I get the following error...
error: no matching function for call to ‘at::Tensor::expand(int, int, int)’
  335 |         auto expanded_state_batch = state_batch.expand(10, -1, -1);
      |                                     ~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~
In file included from /home/iii/tor/m_gym/libtorch/include/ATen/core/Tensor.h:3,
                 from /home/iii/tor/m_gym/libtorch/include/ATen/Tensor.h:3,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/autograd/function_hook.h:3,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/autograd/cpp_hook.h:2,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/autograd/variable.h:6,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/autograd/autograd.h:3,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/api/include/torch/autograd.h:3,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/api/include/torch/all.h:7,
                 from /home/iii/tor/m_gym/libtorch/include/torch/csrc/api/include/torch/torch.h:3,
                 from /home/iii/tor/m_gym/multiv_normal.cpp:2:
/home/iii/tor/m_gym/libtorch/include/ATen/core/TensorBody.h:2372:19: note: candidate: ‘at::Tensor at::Tensor::expand(at::IntArrayRef, bool) const’
 2372 | inline at::Tensor Tensor::expand(at::IntArrayRef size, bool implicit) const {
      |                   ^~~~~~
/home/iii/tor/m_gym/libtorch/include/ATen/core/TensorBody.h:2372:19: note:   candidate expects 2 arguments, 3 provided

It says that .expand only takes two integers but three were given. I've tried a few combinations and I always get an error.
Exactly what I'm doing here is concatenating the 42, 385 tensor ten times into a new tensor. I could do this in a loop with torch::cat, but this would be uglier.

Output:
at::expand expects an at::IntArrayRef the compiler tells you. Hence you want to write something like
auto expanded_state_batch = state_batch.expand({10, -1, -1});


Instruction: What does the "affine" parameter do in PyTorch nn.BatchNorm2d?

Input:
What is the purpose of the affine argument and what does it do?
class DilConv(nn.Module):
    def __init__(self, in_C, out_C, kernel_size, stride, padding, affine=True):
        super(DilConv, self).__init__()
        self.ops = nn.Sequential(
          nn.ReLU(),
          nn.Conv2d(in_C, in_C, kernel_size=kernel_size, stride=stride,
                    padding=padding, dilation=2, groups=in_C, bias=False),
          nn.Conv2d(in_C, out_C, kernel_size=1, bias=False),
          nn.BatchNorm2d(out_C, affine=affine))

    def forward(self, x):
        return self.ops(x)

Output:
If you look at the documentation page for BatchNorm2d, you will read:

affine – a boolean value that when set to True, this module has learnable affine parameters. Default: True

Checking the source code on the base class _NormBase, you will see parameters weight and bias are only defined if the argument affine is set to True. These parameters correspond to gamma and beta in the documentation formulae.


Instruction: How to use pytorch multi-head attention for classification task?

Input: I have a dataset where x shape is (10000, 102, 300) such as ( samples, feature-length, dimension) and y (10000,) which is my binary label. I want to use multi-head attention using PyTorch. I saw the PyTorch documentation from here but there is no explanation of how to use it. How can I use my dataset for classification using multi-head attention?

Output:
I will write a simple pretty code for classification this will work fine, if you need implementation detail then this part is the same as the Encoder layer in Transformer, except in the last you would need a GlobalAveragePooling Layer and a Dense Layer for classification
attention_layer = nn.MultiHeadAttion(300 , 300%num_of_heads==0,dropout=0.1)
neural_net_output = point_wise_neural_network(attention_layer)
normalize = LayerNormalization(input + neural_net_output)
globale_average_pooling = nn.GlobalAveragePooling(normalize)
nn.Linear(input , num_of_classes)(global_average_pooling)


Instruction: Size of WebDataset in Pytorch

Input:
When it comes to the Pytorch Dataloader which takes a default dataset (e.g. datasets.ImageFolder), we can find the size of a dataset that is used by the dataloader with len(dataloader). However, what about WebDataset?
As WebDataset is a PyTorch Dataset, is it possible to get the size of a loader which takes a WebDataset?
https://webdataset.github.io/webdataset/

Output:
WebDataset doesn't provide a __len__ method, as it conforms to the PyTorch IterableDataset interface. IterableDataset is designed for stream-like data, and considers it wrong to have a len().
If you have code that depends on len() to be available, you can set the length to some value using with_length():
>>> dataset = wds.WebDataset(url)
>>> len(dataset)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: object of type 'WebDataset' has no len()
>>> dataset = dataset.with_length(10)
>>> len(dataset)
10


Instruction: Batched Cosine Similarity in PyTorch

Input:
Inputs:

Tensor a of shape [batch_size, n, d]
Tensor b of shape [batch_size, m, d]

Output:

Tensor c of shape [batch_size, n, m] where c[i, j, k] is the cosine similarity between a[i, j] and b[i, k]

How to implement this efficiently in PyTorch (preferably without for loops)?

Output:
try this:
c = torch.cosine_similarity(a.unsqueeze(2), b.unsqueeze(1), dim=-1)


Instruction: What's the difference between torch.mm, torch.matmul and torch.mul?

Input:
After reading the pytorch documentation, I still require help in understanding the difference between torch.mm, torch.matmul and torch.mul. As I do not fully understand them, I cannot concisely explain this.
B = torch.tensor([[ 1.1207],
        [-0.3137],
        [ 0.0700],
        [ 0.8378]])

C = torch.tensor([[ 0.5146,  0.1216, -0.5244,  2.2382]])

print(torch.mul(B,C))

print(torch.matmul(B,C))

print(torch.mm(B,C))

All three produce the following output (i.e. they perform matrix multiplication):
tensor([[ 0.5767,  0.1363, -0.5877,  2.5084],
        [-0.1614, -0.0381,  0.1645, -0.7021],
        [ 0.0360,  0.0085, -0.0367,  0.1567],
        [ 0.4311,  0.1019, -0.4393,  1.8752]])

A = torch.tensor([[1.8351,2.1536], [-0.8320,-1.4578]])
B = torch.tensor([[2.9355, 0.3450], [0.5708, 1.9957]])
print(torch.mul(A,B))
print(torch.matmul(A,B))
print(torch.mm(A,B))

Different outputs are produced. torch.mm no longer performs matrix multiplication (broadcasts and performs element-wise multiplication instead, whilst the other two still perform matrix multiplication.
tensor([[ 5.3869,  0.7430],
        [-0.4749, -2.9093]])
tensor([[ 6.6162,  4.9310],
        [-3.2744, -3.1964]])
tensor([[ 6.6162,  4.9310],
        [-3.2744, -3.1964]])

Inputs
tensor1 = torch.randn(10, 3, 4)
tensor2 = torch.randn(4)

tensor1 =
tensor([[[-0.2267,  0.6311, -0.5689,  1.2712],
         [-0.0241, -0.5362,  0.5481, -0.4534],
         [-0.9773, -0.6842,  0.6927,  0.3363]],

        [[-2.6759,  0.7817,  2.6821,  0.7037],
         [ 0.1804,  0.3938, -1.2235,  0.8729],
         [-1.9873, -0.5030,  0.0945,  0.2688]],

        [[ 0.4244,  1.7350,  0.0558, -0.1861],
         [-0.9063, -0.4737, -0.4284, -0.3883],
         [ 0.4827, -0.2628,  1.0084,  0.2769]],

        [[ 0.2939,  0.4604,  0.8014, -1.8760],
         [ 1.8807,  0.1623,  0.2344, -0.6221],
         [ 1.3964,  3.1637,  0.7889,  0.1195]],

        [[-0.7202,  1.4250,  2.4302,  1.4811],
         [-0.2301,  0.6280,  0.5379,  0.5178],
         [-2.1073, -1.4399, -0.9451,  0.8534]],

        [[ 2.8178, -0.4451, -0.7871, -0.5198],
         [ 0.2825,  1.0692,  0.1559,  1.2945],
         [-0.5828, -1.6287, -2.0661, -0.4107]],

        [[ 0.5077, -0.6349, -0.0160, -0.4477],
         [-0.8070,  0.3746,  1.1852,  0.0351],
         [-0.6454,  1.5877,  0.8561,  1.1021]],

        [[ 0.1191,  1.0116,  0.5807,  1.2105],
         [-0.5403,  1.2404,  1.1532,  0.6537],
         [ 1.4757, -1.3648, -1.7158, -1.0289]],

        [[-0.1326,  0.3715,  0.2429, -0.0794],
         [ 0.3224, -0.3064,  0.1963,  0.7276],
         [ 0.9098,  1.5984, -1.4953,  0.0420]],

        [[ 0.1511,  0.9691, -0.5204,  0.3858],
         [ 0.4566,  1.5482, -0.3401,  0.5960],
         [-0.9998,  0.7198,  0.9286,  0.4498]]])

tensor2 =
tensor([-1.6350,  1.0335, -0.9023,  0.0696])

print(torch.mul(tensor1,tensor2))
print(torch.matmul(tensor1,tensor2))
print(torch.mm(tensor1,tensor2))

Outputs are all different. I think torch.mul broadcasts and multiplies every 4 elements of the matrix by the vector, tensor2, i.e. [-0.2267,  0.6311, -0.5689,  1.2712] x tensor 2  element-wise, [-0.0241, -0.5362,  0.5481, -0.4534] x tensor 2 element-wise and so on. I do not understand what  torch.matmul is doing. I think it is to do with the 5th bullet-point of the documentation (If both arugments...), but I am unable to make sense of this. https://pytorch.org/docs/stable/generated/torch.matmul.html
I think the reason torch.mm is unable to produce an output is the fact that it cannot broadcast (please correct me if I'm wrong).
tensor([[[ 3.7071e-01,  6.5221e-01,  5.1335e-01,  8.8437e-02],
         [ 3.9400e-02, -5.5417e-01, -4.9460e-01, -3.1539e-02],
         [ 1.5979e+00, -7.0715e-01, -6.2499e-01,  2.3398e-02]],

        [[ 4.3752e+00,  8.0790e-01, -2.4201e+00,  4.8957e-02],
         [-2.9503e-01,  4.0699e-01,  1.1040e+00,  6.0723e-02],
         [ 3.2494e+00, -5.1981e-01, -8.5253e-02,  1.8701e-02]],

        [[-6.9397e-01,  1.7931e+00, -5.0379e-02, -1.2945e-02],
         [ 1.4818e+00, -4.8954e-01,  3.8657e-01, -2.7010e-02],
         [-7.8920e-01, -2.7163e-01, -9.0992e-01,  1.9265e-02]],

        [[-4.8055e-01,  4.7582e-01, -7.2309e-01, -1.3051e-01],
         [-3.0750e+00,  1.6770e-01, -2.1146e-01, -4.3281e-02],
         [-2.2832e+00,  3.2697e+00, -7.1183e-01,  8.3139e-03]],

        [[ 1.1775e+00,  1.4727e+00, -2.1928e+00,  1.0304e-01],
         [ 3.7617e-01,  6.4900e-01, -4.8534e-01,  3.6025e-02],
         [ 3.4455e+00, -1.4882e+00,  8.5277e-01,  5.9369e-02]],

        [[-4.6072e+00, -4.6005e-01,  7.1024e-01, -3.6160e-02],
         [-4.6191e-01,  1.1051e+00, -1.4067e-01,  9.0053e-02],
         [ 9.5283e-01, -1.6833e+00,  1.8643e+00, -2.8571e-02]],

        [[-8.3005e-01, -6.5622e-01,  1.4461e-02, -3.1148e-02],
         [ 1.3195e+00,  3.8716e-01, -1.0694e+00,  2.4421e-03],
         [ 1.0553e+00,  1.6409e+00, -7.7250e-01,  7.6669e-02]],

        [[-1.9477e-01,  1.0455e+00, -5.2398e-01,  8.4209e-02],
         [ 8.8343e-01,  1.2820e+00, -1.0405e+00,  4.5478e-02],
         [-2.4128e+00, -1.4106e+00,  1.5482e+00, -7.1578e-02]],

        [[ 2.1675e-01,  3.8391e-01, -2.1914e-01, -5.5219e-03],
         [-5.2707e-01, -3.1668e-01, -1.7711e-01,  5.0619e-02],
         [-1.4876e+00,  1.6520e+00,  1.3493e+00,  2.9198e-03]],

        [[-2.4706e-01,  1.0015e+00,  4.6955e-01,  2.6842e-02],
         [-7.4663e-01,  1.6001e+00,  3.0685e-01,  4.1462e-02],
         [ 1.6347e+00,  7.4395e-01, -8.3792e-01,  3.1291e-02]]])
tensor([[ 1.6247, -1.0409,  0.2891],
        [ 2.8120,  1.2767,  2.6630],
        [ 1.0358,  1.3518, -1.9515],
        [-0.8583, -3.1620,  0.2830],
        [ 0.5605,  0.5759,  2.8694],
        [-4.3932,  0.5925,  1.1053],
        [-1.5030,  0.6397,  2.0004],
        [ 0.4109,  1.1704, -2.3467],
        [ 0.3760, -0.9702,  1.5165],
        [ 1.2509,  1.2018,  1.5720]])

Output:
In short:

torch.mm - performs a matrix multiplication without broadcasting - (2D tensor) by (2D tensor)
torch.mul - performs a elementwise multiplication with broadcasting - (Tensor) by (Tensor or Number)
torch.matmul - matrix product with broadcasting - (Tensor) by (Tensor) with different behaviors depending on the tensor shapes (dot product, matrix product, batched matrix products).

Some details:

torch.mm - performs a matrix multiplication without broadcasting

It expects two 2D tensors so n×m * m×p = n×p
From the documentation https://pytorch.org/docs/stable/generated/torch.mm.html:
This function does not broadcast. For broadcasting matrix products, see torch.matmul().

torch.mul - performs a elementwise multiplication with broadcasting - (Tensor) by (Tensor or Number)

Docs: https://pytorch.org/docs/stable/generated/torch.mul.html
torch.mul does not perform a matrix multiplication. It broadcasts two tensors and performs an elementwise multiplication. So when you uses it with tensors 1x4 * 4x1 it will work similar to:
import torch

a = torch.FloatTensor([[1], [2], [3]])
b = torch.FloatTensor([[1, 10, 100]])
a, b = torch.broadcast_tensors(a, b)
print(a)
print(b)
print(a * b)

tensor([[1., 1., 1.],
        [2., 2., 2.],
        [3., 3., 3.]])
tensor([[  1.,  10., 100.],
        [  1.,  10., 100.],
        [  1.,  10., 100.]])
tensor([[  1.,  10., 100.],
        [  2.,  20., 200.],
        [  3.,  30., 300.]])

torch.matmul

It is better to check out the official documentation https://pytorch.org/docs/stable/generated/torch.matmul.html as it uses different modes depending on the input tensors. It may perform dot product, matrix-matrix product or batched matrix products with broadcasting.
As for your question regarding product of:
tensor1 = torch.randn(10, 3, 4)
tensor2 = torch.randn(4)

it is a batched version of a product. please check this simple example for understanding:
import torch

# 3x1x3
a = torch.FloatTensor([[[1, 2, 3]], [[3, 4, 5]], [[6, 7, 8]]])
# 3
b = torch.FloatTensor([1, 10, 100])
r1 = torch.matmul(a, b)

r2 = torch.stack((
    torch.matmul(a[0], b),
    torch.matmul(a[1], b),
    torch.matmul(a[2], b),
))
assert torch.allclose(r1, r2)

So it can be seen as a multiple operations stacked together across batch dimension.
Also it may be useful to read about broadcasting:
https://pytorch.org/docs/stable/notes/broadcasting.html#broadcasting-semantics


Instruction: Pytorch low gpu util after first epoch

Input:
Hi I'm training my pytorch model on remote server.
All the job is managed by slurm.
My problem is 'training is extremely slower after training first epoch.'
I checked gpu utilization.
On my first epoch, utilization was like below image.

I can see gpu was utilized.
But from second epoch utilized percentage is almos zero

My dataloader code like this
class img2selfie_dataset(Dataset):
    def __init__(self, path, transform, csv_file, cap_vec):
        self.path = path
        self.transformer = transform
        self.images = [path + item for item in list(csv_file['file_name'])]
        self.smiles_list = cap_vec

    def __getitem__(self, idx):
        img = Image.open(self.images[idx])
        img = self.transformer(img)
        label = self.smiles_list[idx]
        label = torch.Tensor(label)
        return img, label.type(torch.LongTensor)

    def __len__(self):
        return len(self.images)

My dataloader is defined like this
train_data_set = img2selfie_dataset(train_path, preprocess, train_dataset, train_cap_vec)
train_loader = DataLoader(train_data_set, batch_size = 256, num_workers = 2, pin_memory = True)
val_data_set = img2selfie_dataset(train_path, preprocess, val_dataset, val_cap_vec)
val_loader = DataLoader(val_data_set, batch_size = 256, num_workers = 2, pin_memory = True)

My training step defined like this
train_loss = []
valid_loss = []
epochs = 20
best_loss = 1e5
for epoch in range(1, epochs + 1):
    print('Epoch {}/{}'.format(epoch, epochs))
    print('-' * 10)
    epoch_train_loss, epoch_valid_loss = train(encoder_model, transformer_decoder, train_loader, val_loader, criterion, optimizer)
    train_loss.append(epoch_train_loss)
    valid_loss.append(epoch_valid_loss)
    if len(valid_loss) > 1:
        if valid_loss[-1] < best_loss:
            print(f"valid loss on this {epoch} is better than previous one, saving model.....")
            torch.save(encoder_model.state_dict(), 'model/encoder_model.pickle')
            torch.save(transformer_decoder.state_dict(), 'model/decoder_model.pickle')
            best_loss = valid_loss[-1]
            print(best_loss)
    print(f'Epoch : [{epoch}] Train Loss : [{train_loss[-1]:.5f}], Valid Loss : [{valid_loss[-1]:.5f}]')

In my opinion, if this problem comes from my code. It wouldn't have hitted 100% utilization in first epoch.

Output:
I fixed this issue with moving my training data into local drive.
My remote server(school server) policy was storing personel data into NAS.
And file i/o from NAS proveked heavy load on network.
It was also affected by other user's file i/o from NAS.
After I moved training data into NAS, everything is fine.


Instruction: How to convert a PyTorch nn.Module into a HuggingFace PreTrainedModel object?

Input:
Given a simple neural net in Pytorch like:
import torch.nn as nn

net = nn.Sequential(
      nn.Linear(3, 4),
      nn.Sigmoid(),
      nn.Linear(4, 1),
      nn.Sigmoid()
      ).to(device)

How do I convert it into a Huggingface PreTrainedModel object?
The goal is to convert the Pytorch nn.Module object from nn.Sequential into the Huggingface PreTrainedModel object, then run something like:
import torch.nn as nn
from transformers.modeling_utils import PreTrainedModel

net = nn.Sequential(
      nn.Linear(3, 4),
      nn.Sigmoid(),
      nn.Linear(4, 1),
      nn.Sigmoid()
      ).to(device)

# Do something to convert the Pytorch nn.Module to the PreTrainedModel object.
shiny_model = do_some_magic(net, some_args, some_kwargs)

# Save the shiny model that is a `PreTrainedModel` object.
shiny_model.save_pretrained("shiny-model")

PreTrainedModel.from_pretrained("shiny-model")

And it seems like to build/convert any native Pytorch models into a Huggingface one, there's a need for some configurations https://huggingface.co/docs/transformers/main_classes/configuration
There are many how-tos to train models "from scratch", e.g.

[Using BertLMHeadModel, not that scratch] https://www.kaggle.com/code/mojammel/train-model-from-scratch-with-huggingface/notebook (this is also fine-tuning from bert, not scratch)

[Not really scratch, using roberta as template] https://huggingface.co/blog/how-to-train (this is fine-tuning from roberta, not really training from scratch)

[Sort of uses some Config template] https://www.thepythoncode.com/article/pretraining-bert-huggingface-transformers-in-python (this is kinda from scratch but uses the template from BERT to generate the config, what if we want to change how the model works, how should the config look like?)

[Kinda defined a template but using RobertaForMaskedLM] https://skimai.com/roberta-language-model-for-spanish/ (this looks like it kinda defines a template but restricts it to RobertaForMaskedLM template)

Questions in parts:

If we have a much simpler Pytorch model like in the code snippet above, how to create a Pretrained Model from scratch in Huggingface?

How to create the Pretrained model config we need for Huggingface to make the converting from native Pytorch nn.Module work?

Output:
You will need to define custom configuration and custom model classes. It is important to define attributes model_type and config_class inside those classes:
import torch.nn as nn
from transformers import PreTrainedModel, PretrainedConfig
from transformers import AutoModel, AutoConfig

class MyConfig(PretrainedConfig):
    model_type = 'mymodel'
    def __init__(self, important_param=42, **kwargs):
        super().__init__(**kwargs)
        self.important_param = important_param

class MyModel(PreTrainedModel):
    config_class = MyConfig
    def __init__(self, config):
        super().__init__(config)
        self.config = config
        self.model = nn.Sequential(
                          nn.Linear(3, self.config.important_param),
                          nn.Sigmoid(),
                          nn.Linear(self.config.important_param, 1),
                          nn.Sigmoid()
                          )
    def forward(self, input):
        return self.model(input)

Now you can create (and obviously train a new model), save and then load your model locally
config = MyConfig(4)
model = MyModel(config)
model.save_pretrained('./my_model_dir')

new_model = MyModel.from_pretrained('./my_model_dir')
new_model

If you wish to use AutoModel, you will have to register your classes:
AutoConfig.register("mymodel", MyConfig)
AutoModel.register(MyConfig, MyModel)

new_model = AutoModel.from_pretrained('./my_model_dir')
new_model


Instruction: How to use CTC Loss Seq2Seq correctly?

Input:
I am trying to create ASR model by myself and learn how to use CTC loss.
I test and I see this:
ctc_loss = nn.CTCLoss(blank=95)

output: tensor([[63,  8,  1, 38, 29, 14, 41, 71, 14, 29, 45, 41, 3]]): torch.Size([1, 13]); output_size: tensor([13])

input1: torch.Size([167, 1, 96]); input1_size: tensor([167])

After applying the argmax on this input (= prediction of phonems)
torch.argmax(input1, dim=2)

I get a series of symbols:
tensor([[63, 63, 63, 63, 63, 63, 95, 95, 63, 63, 95, 95,  8,  8,  8, 95,  8, 95,
           8,  8, 95, 95, 95,  1,  1, 95,  1, 95,  1,  1, 95, 95, 38, 95, 95, 38,
          38, 38, 38, 38, 29, 29, 29, 29, 29, 29, 29, 95, 29, 29, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 14, 95, 14, 95, 95, 95, 95, 14, 95, 14, 41, 41,
          41, 95, 41, 41, 41, 41, 41, 41, 71, 71, 71, 95, 71, 71, 71, 71, 71, 95,
          95, 14, 14, 95, 14, 14, 95, 14, 14, 95, 29, 29, 95, 29, 29, 29, 29, 29,
          29, 29, 45, 95, 95, 45, 45, 95, 45, 45, 45, 45, 41, 95, 41, 41, 95, 95,
          95, 41, 41, 41,  3,  3,  3,  3,  3, 95,  3,  3,  3, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95]])

and the following loss values.
ctc_loss(input1, output, input_size, output_size)
# Returns 222.8446

With a different input:
input2: torch.Size([167, 1, 96]) input2_size: tensor([167])

torch.argmax(input2, dim=2)

the prediction is just a sequence of blank symbols.
tensor([[95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, 95,
          95, 95, 95, 95, 95]])

However, the loss value with the same desired output is much lower.
ctc_loss(input2, output, input_size, output_size)
# Returns 3.7955

I don't know why input1 is better than input2 but the loss of input1 is higher than input2? Can someone explain that?

Output:
The CTC loss does not operate on the argmax predictions but on the entire output distribution. The CTC loss is the sum of the negative log-likelihood of all possible output sequences that produce the desired output. The output symbols might be interleaved with the blank symbols, which leaves exponentially many possibilities. It is, in theory, possible that the sum of the negative log-likelihoods of the correct outputs is low and still the most probable sequence is all blanks.
In practice, this is quite rare, so I guess there might be a problem somewhere else. The CTCLoss as implemented in PyTorch requires log probabilities as the input that you get, e.g., by applying the log_softmax function. Different sorts of input might lead to strange results such the one you observe.


Instruction: Multiclass classification, IndexError: Target 2 is out of bounds

Input:
I am facing a multiclass classification problem related to the activity of some drugs using Pytorch neural net, I have three activity classes (0, 1 and 2), to tackle the problem I adopted the one vs. one approach, thus creating three binary classifiers: 0 vs. 1, 1 vs. 2 and 2 vs. 0. When I train the second classifier (class 1 vs. class 2) I get the following error:
IndexError: Target 2 is out of bounds.

Is there a method to solve it without reassigning labels? Thank you all!
This is my net, is a Graph Isomorphism Network build with Pytorch Geometric:
class GIN1(torch.nn.Module):
  def __init__(self, h):
     super(GIN1, self).__init__()
     dim_h_conv = h
     dim_h_fc = dim_h_conv*5

     # Convolutional layers
     self.conv1 = GINConv(Sequential(Linear(14, dim_h_conv),
                                     BatchNorm1d(dim_h_conv), ReLU(),
                                     Linear(dim_h_conv, dim_h_conv), ReLU()))
     self.conv2 = GINConv(Sequential(Linear(dim_h_conv, dim_h_conv),
                                     BatchNorm1d(dim_h_conv), ReLU(),
                                     Linear(dim_h_conv, dim_h_conv), ReLU()))
     self.conv3 = GINConv(Sequential(Linear(dim_h_conv, dim_h_conv),
                                     BatchNorm1d(dim_h_conv), ReLU(),
                                     Linear(dim_h_conv, dim_h_conv), ReLU()))
     self.conv4 = GINConv(Sequential(Linear(dim_h_conv, dim_h_conv),
                                     BatchNorm1d(dim_h_conv), ReLU(),
                                     Linear(dim_h_conv, dim_h_conv), ReLU()))
     self.conv5 = GINConv(Sequential(Linear(dim_h_conv, dim_h_conv),
                                     BatchNorm1d(dim_h_conv), ReLU(),
                                     Linear(dim_h_conv, dim_h_conv), ReLU()))

     # Fully connected layers
     self.lin1 = Linear(dim_h_fc, dim_h_fc)
     self.lin2 = Linear(dim_h_fc, 2)

     self.initialize_w()

  def forward(self, x, edge_index, batch):
    h1 = self.conv1(x, edge_index)
    h2 = self.conv2(h1, edge_index)
    h3 = self.conv3(h2, edge_index)
    h4 = self.conv4(h3, edge_index)
    h5 = self.conv5(h4, edge_index)

    # Graph level readout
    h1 = global_add_pool(h1, batch)
    h2 = global_add_pool(h2, batch)
    h3 = global_add_pool(h3, batch)
    h4 = global_add_pool(h4, batch)
    h5 = global_add_pool(h5, batch)

    # Concatenate graph embeddings
    h = torch.cat((h1, h2, h3, h4, h5), dim=1)

    # Classifier
    h = self.lin1(h)
    h = h.relu()
    h = F.dropout(h, p=hp_gin1['p'], training=self.training)
    h = self.lin2(h)
    h = F.log_softmax(h, dim=1)

    return h

  def initialize_w(self):
    for m in self.modules():

      if isinstance(m, Linear):
        torch.nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')
        torch.nn.init.constant_(m.bias, 0)

      if isinstance(m, BatchNorm1d):
        torch.nn.init.constant_(m.weight, 1)
        torch.nn.init.constant_(m.bias, 0)

And this is my training loop:
gin2 = GIN2(h=hp_gin2['h']) #40
optimizer = torch.optim.Adam(gin2.parameters(), lr=hp_gin2['lr'])
criterion = torch.nn.CrossEntropyLoss()

def train(train_loader):
  gin2.train()
  loss_all = 0
  for data in train_loader:
    output = gin2(data.x, data.edge_index, data.batch)
    loss = criterion(output, data.y)

    l2_lambda = hp_gin2['lambda']
    l2_norm = sum(p.pow(2.0).sum()
                 for p in gin2.parameters())
    loss = loss + l2_lambda * l2_norm

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    loss_all += loss.item() * data.num_graphs
  return loss_all / len(train_loader.dataset)

def test_loss(loader):
  total_loss_val = 0
  with torch.no_grad():
   for data in loader:
     output = gin2(data.x, data.edge_index, data.batch)
     batch_loss = criterion(output, data.y)

     total_loss_val += batch_loss.item() * data.num_graphs
  return total_loss_val / len(loader.dataset)

def test(loader):
  gin2.eval()
  correct = 0
  for data in loader:
    output = gin2(data.x, data.edge_index, data.batch)

    accuracy = Accuracy(average='macro', num_classes=2)
    acc = accuracy(output, data.y)

  return acc

Output: OP needed to match the output dimension of their model with the number of label classes (see discussion).


Instruction: YOLOv7 RuntimeError: CUDA error: unknown error

Input:
Getting the following error with my current setup after following this guide when trying to train a custom model for YOLOv7: https://docs.nvidia.com/cuda/wsl-user-guide/index.html.
OS: Ubuntu (Windows 10 WSL)
Hardware: 16gb RAM, RTX 3070
Python Version: 3.8.10
Driver Version: 517.48
PyTorch Version:
>>> import torch
>>> print(torch.__version__)
1.10.0a0+3fd9dcf
>>> import torchvision
>>> print(torchvision.__version__)
0.11.0a0

Cuda Version:
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2021 NVIDIA Corporation
Built on Wed_Jul_14_19:41:19_PDT_2021
Cuda compilation tools, release 11.4, V11.4.100
Build cuda_11.4.r11.4/compiler.30188945_0

Some common issues that have solved this issue for other people that I've attempted:

Restarting the computer
apt-get install nvidia-modprobe (https://github.com/pytorch/pytorch/issues/49081)

Docker Container:
sudo docker run --name yolov7 --gpus all -it -v "/mnt/c/coco/":"/coco/" -v "/mnt/c/yolov7/":"/yolov7/" --shm-size=16gb nvcr.io/nvidia/pytorch:21.08-py3
The part that I don't understand is that torch appears to be operating correctly:
>>> import torch
>>> print(torch.cuda.current_device())
0
>>> torch.rand(1)
tensor([0.3052])

nvidia-smi
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 515.76.02    Driver Version: 517.48       CUDA Version: 11.7     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  On   | 00000000:2B:00.0  On |                  N/A |
|  0%   46C    P8    27W / 240W |    423MiB /  8192MiB |      1%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

From Within Docker Container:
python train.py --workers 1 --device 0 --batch-size 2 --data data/coco.yaml --img-size 1920 --cfg cfg/training/yolov7.yaml --weights 'yolov7_training.pt' --name yolov7 --hyp data/hyp.scratch.custom.yaml
The above call produces the following error:
Traceback (most recent call last):
  File "train.py", line 616, in <module>
    train(hyp, opt, device, tb_writer)
  File "train.py", line 361, in train
    pred = model(imgs)  # forward
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1056, in _call_impl
    return forward_call(*input, **kwargs)
  File "/yolov7/models/yolo.py", line 599, in forward
    return self.forward_once(x, profile)  # single-scale inference, train
  File "/yolov7/models/yolo.py", line 625, in forward_once
    x = m(x)  # run
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1056, in _call_impl
    return forward_call(*input, **kwargs)
  File "/yolov7/models/common.py", line 108, in forward
    return self.act(self.bn(self.conv(x)))
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1056, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/activation.py", line 395, in forward
    return F.silu(input, inplace=self.inplace)
  File "/opt/conda/lib/python3.8/site-packages/torch/nn/functional.py", line 1901, in silu
    return torch._C._nn.silu(input)
RuntimeError: CUDA error: unknown error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

Output:
Even on WSL, your Host is Windows and will have the same limitations.
On windows there is a watchdog that kills too long kernels (WDDM).
If your NVidia GPU is a secondary GPU, try to disable WDDM TDR.
If it's your primary GPU, you can experiment screen freezes when you use pyTorch.
there is some informations on how to disable WDDM in this post : https://stackoverflow.com/a/13185441/4866974


Instruction: filter out rows which satisfy a condition in each column

Input:
Suppose I have a tensor:
input: ([[-0.5535,  0.0000],
        [ 0.0000,  0.0000],
        [-1.1370, -0.2736],
        [-1.2300,  0.9185]])

Output:([[-0.5535,  0.0000],
        [-1.1370, -0.2736],
        [-1.2300,  0.9185]])

I need to keep only the rows which have non-zero elements in all columns, and the index of the deleted row. For simplicity, I have limited the matrix to two columns, however in my case the number of columns and rows keeps changing in every iteration.
I have found solutions where the condition may satisfy any element in the matrix, or there may be separate conditions to satisfy per column, but I couldn't figure out how to solve this particular case.
Thank you.

Output:
I will answer my own question since I found the solution in pytorch.
This function will return the row-indices of non
x[torch.nonzero(torch.tensor(x), as_tuple=True)[0].unique()]
OR
x[torch.nonzero(torch.tensor(x.sum(1)), as_tuple=True)[0]]


Instruction: How to do multiple forward pass and one backward pass pytorch?

Input:
import torch
import torchvision.models as models
model = models.resnet18()
optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)
x = torch.randn(1, 3, 224, 224)
y = torch.randn(1, 3, 224, 224)

#1st Approach
loss1 = model(x).mean()
loss2 = model(y).mean()
(loss1+loss2).backward()
optimizer.step()

I want to forward two datasets and their total loss will be used for backward and update one model. Is this approach correct?
#2nd Approach
loss1 = model(x).mean()
loss1.backward()
loss2 = model(y).mean()
loss2.backward()
optimizer.step()

And what is the difference between the first and second approachs ?

Output:
Both of them are actually equivalent: The gradient gets acccumulated additively in the backpropagation (which is a convenient implementation for nodes that appear multiple times in the computation graph). So both of them are pretty much identical.
But to make the code readable and really make obvious what is happening, I would prefer the first approach. The second method (as described above) is basically "abusing" that effect of accumulating gradients - it is not actually abuse but it is way more common, and as I said in my opinion way easier to read to use the first way.


Instruction: PyTorch 1.12 on Mac Monterey

Input:
I cannot use PyTorch 1.12.1 on macOS 12.6 Monterey with M1 chip.
Tried to install and run from Python 3.8, 3.9 and 3.10 with the same result.
I think that PyTorch was working before I updated macOS to Monterey. And the Rust bindings, tch-rs are still working.
Here is my install and the error messages I get when trying to run.
Install
brew install libtorch

python3.9 -m venv venv39

source venv39/bin/activate

pip3 install torch torchvision torchaudio

Error message
python
Python 3.9.14 (main, Sep  6 2022, 23:16:16)
[Clang 13.1.6 (clang-1316.0.21.2.5)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "~/Documents/install/Modern_Computer_Vision/venv39/lib/python3.9/site-packages/torch/__init__.py", line 202, in <module>
    from torch._C import *  # noqa: F403
ImportError: dlopen(~/Documents/install/Modern_Computer_Vision/venv39/lib/python3.9/site-packages/torch/_C.cpython-39-darwin.so, 0x0002): Symbol not found: (__ZN4c10d11debug_levelEv)
  Referenced from: '@/Documents/install/Modern_Computer_Vision/venv39/lib/python3.9/site-packages/torch/lib/libtorch_python.dylib'
  Expected in: '/opt/homebrew/Cellar/libtorch/1.12.1/lib/libtorch_cpu.dylib'

Tried using Miniconda
I had almost the same result.
conda create -n conda39 python=3.9 -y
conda activate conda39
conda install pytorch torchvision torchaudio -c pytorch

❯ python
Python 3.9.12 (main, Apr  5 2022, 01:52:34)
[Clang 12.0.0 ] :: Anaconda, Inc. on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/sami/miniconda3/lib/python3.9/site-packages/torch/__init__.py", line 202, in <module>
    from torch._C import *  # noqa: F403
ImportError: dlopen(/Users/sami/miniconda3/lib/python3.9/site-packages/torch/_C.cpython-39-darwin.so, 0x0002): Symbol not found: (__ZN4c10d11debug_levelEv)
  Referenced from: '/Users/sami/miniconda3/lib/python3.9/site-packages/torch/lib/libtorch_python.dylib'
  Expected in: '/opt/homebrew/Cellar/libtorch/1.12.1/lib/libtorch_cpu.dylib'

Output: I recommend not touching your system python installations for your own projects, instead the recommended way is using conda (see here). The reason is that each conda environment encapsulates a whole separate python installation that does not interfere (and doesn't get interfered with) with any other programs. This is especially important for C/C++ libraries like the ones pytorch is using.