Instruction: Skorch RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

Input:
I'm trying to develop an image segmentation model. In the below code I keep hitting a RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same. I'm not sure why as I've tried to load both my data and my UNet model to the GPU using .cuda() (although not the skorch model-- not sure how to do that). I'm using a library for active learning, modAL, which wraps skorch.
from modAL.models import ActiveLearner
import numpy as np
import torch

from torch import nn
from torch import Tensor
from torch.utils.data import DataLoader
from torch.utils.data import Dataset

from skorch.net import NeuralNet

from modAL.models import ActiveLearner
from modAL.uncertainty import classifier_uncertainty, classifier_margin
from modAL.utils.combination import make_linear_combination, make_product
from modAL.utils.selection import multi_argmax
from modAL.uncertainty import uncertainty_sampling

from model import UNet
from skorch.net import NeuralNet
from skorch.helper import predefined_split
from torch.optim import SGD

import cv2

# Map style dataset,
class ImagesDataset(Dataset):
    """Constructs dataset of satellite images + masks"""
    def __init__(self, image_paths):
        super().__init__()
        self.image_paths = image_paths

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        print("idx:", idx)
        sample_dir = self.image_paths[idx]
        img_path = sample_dir +"/images/"+ Path(sample_dir).name +'.png'
        mask_path = sample_dir +'/mask.png'
        img, mask = cv2.imread(img_path), cv2.imread(mask_path)
        print("shape of img", img.shape)
        return img, mask

# turn data into dataset
train_ds = ImagesDataset(train_dirs)
val_ds = ImagesDataset(valid_dirs)

train_loader = torch.utils.data.DataLoader(train_ds, batch_size=3, shuffle=True, pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_ds, batch_size=1, shuffle=True, pin_memory=True)

# make sure data loaded in cuda for train, validation
for i, (tr, val) in enumerate(train_loader):
    tr, val = tr.cuda(), val.cuda()

for i, (tr2, val2) in enumerate(val_loader):
    tr2, val2 = tr2.cuda(), val2.cuda()

X, y = next(iter(train_loader))
X_train = np.array(X.reshape(3,3,1024,1024))
y_train = np.array(y.reshape(3,3,1024,1024))

X2, y2 = next(iter(val_loader))
X_test = np.array(X2.reshape(1,3,1024,1024))
y_test = np.array(y2.reshape(1,3,1024,1024))

module = UNet(pretrained=True)
if torch.cuda.is_available():
    module = module.cuda()

# create the classifier

net = NeuralNet(
    module,
    criterion=torch.nn.NLLLoss,
    batch_size=32,
    max_epochs=20,
    optimizer=SGD,
    optimizer__momentum=0.9,
    iterator_train__shuffle=True,
    iterator_train__num_workers=4,
    iterator_valid__shuffle=False,
    iterator_valid__num_workers=4,
    train_split=predefined_split(val_ds),
    device='cuda',
)

# assemble initial data
n_initial = 1
initial_idx = np.random.choice(range(len(X_train)), size=n_initial, replace=False)
X_initial = X_train[initial_idx]
y_initial = y_train[initial_idx]

# generate the pool, remove the initial data from the training dataset
X_pool = np.delete(X_train, initial_idx, axis=0)
y_pool = np.delete(y_train, initial_idx, axis=0)

# train the activelearner
# shape of 4D matrix is 'batch', 'channel', 'width', 'height')
learner = ActiveLearner(
    estimator= net,
    X_training=X_initial, y_training=y_initial,
)

The full error trace is:
    ---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-83-0af6007b6b72> in <module>
      8 learner = ActiveLearner(
      9     estimator= net,
---> 10     X_training=X_initial, y_training=y_initial,
     11     # X_training=X_initial, y_training=y_initial,
     12 )

~/.local/lib/python3.7/site-packages/modAL/models/learners.py in __init__(self, estimator, query_strategy, X_training, y_training, bootstrap_init, on_transformed, **fit_kwargs)
     80                  ) -> None:
     81         super().__init__(estimator, query_strategy,
---> 82                          X_training, y_training, bootstrap_init, on_transformed, **fit_kwargs)
     83
     84     def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: bool = False, **fit_kwargs) -> None:

~/.local/lib/python3.7/site-packages/modAL/models/base.py in __init__(self, estimator, query_strategy, X_training, y_training, bootstrap_init, on_transformed, force_all_finite, **fit_kwargs)
     70         self.y_training = y_training
     71         if X_training is not None:
---> 72             self._fit_to_known(bootstrap=bootstrap_init, **fit_kwargs)
     73             self.Xt_training = self.transform_without_estimating(self.X_training) if self.on_transformed else None
     74

~/.local/lib/python3.7/site-packages/modAL/models/base.py in _fit_to_known(self, bootstrap, **fit_kwargs)
    160         """
    161         if not bootstrap:
--> 162             self.estimator.fit(self.X_training, self.y_training, **fit_kwargs)
    163         else:
    164             n_instances = self.X_training.shape[0]

~/.local/lib/python3.7/site-packages/skorch/net.py in fit(self, X, y, **fit_params)
    901             self.initialize()
    902
--> 903         self.partial_fit(X, y, **fit_params)
    904         return self
    905

~/.local/lib/python3.7/site-packages/skorch/net.py in partial_fit(self, X, y, classes, **fit_params)
    860         self.notify('on_train_begin', X=X, y=y)
    861         try:
--> 862             self.fit_loop(X, y, **fit_params)
    863         except KeyboardInterrupt:
    864             pass

~/.local/lib/python3.7/site-packages/skorch/net.py in fit_loop(self, X, y, epochs, **fit_params)
    774
    775             self.run_single_epoch(dataset_train, training=True, prefix="train",
--> 776                                   step_fn=self.train_step, **fit_params)
    777
    778             if dataset_valid is not None:

~/.local/lib/python3.7/site-packages/skorch/net.py in run_single_epoch(self, dataset, training, prefix, step_fn, **fit_params)
    810             yi_res = yi if not is_placeholder_y else None
    811             self.notify("on_batch_begin", X=Xi, y=yi_res, training=training)
--> 812             step = step_fn(Xi, yi, **fit_params)
    813             self.history.record_batch(prefix + "_loss", step["loss"].item())
    814             self.history.record_batch(prefix + "_batch_size", get_len(Xi))

~/.local/lib/python3.7/site-packages/skorch/net.py in train_step(self, Xi, yi, **fit_params)
    707             return step['loss']
    708
--> 709         self.optimizer_.step(step_fn)
    710         return step_accumulator.get_step()
    711

~/.local/lib/python3.7/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     24         def decorate_context(*args, **kwargs):
     25             with self.__class__():
---> 26                 return func(*args, **kwargs)
     27         return cast(F, decorate_context)
     28

~/.local/lib/python3.7/site-packages/torch/optim/sgd.py in step(self, closure)
     84         if closure is not None:
     85             with torch.enable_grad():
---> 86                 loss = closure()
     87
     88         for group in self.param_groups:

~/.local/lib/python3.7/site-packages/skorch/net.py in step_fn()
    703         def step_fn():
    704             self.optimizer_.zero_grad()
--> 705             step = self.train_step_single(Xi, yi, **fit_params)
    706             step_accumulator.store_step(step)
    707             return step['loss']

~/.local/lib/python3.7/site-packages/skorch/net.py in train_step_single(self, Xi, yi, **fit_params)
    643         """
    644         self.module_.train()
--> 645         y_pred = self.infer(Xi, **fit_params)
    646         loss = self.get_loss(y_pred, yi, X=Xi, training=True)
    647         loss.backward()

~/.local/lib/python3.7/site-packages/skorch/net.py in infer(self, x, **fit_params)
   1046             x_dict = self._merge_x_and_fit_params(x, fit_params)
   1047             return self.module_(**x_dict)
-> 1048         return self.module_(x, **fit_params)
   1049
   1050     def _get_predict_nonlinearity(self):

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/al/model.py in forward(self, x)
     51
     52     def forward(self, x):
---> 53         conv1 = self.conv1(x)
     54         conv2 = self.conv2(conv1)
     55         conv3 = self.conv3(conv2)

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/.local/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    115     def forward(self, input):
    116         for module in self:
--> 117             input = module(input)
    118         return input
    119

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/.local/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    421
    422     def forward(self, input: Tensor) -> Tensor:
--> 423         return self._conv_forward(input, self.weight)
    424
    425 class Conv3d(_ConvNd):

~/.local/lib/python3.7/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    418                             _pair(0), self.dilation, self.groups)
    419         return F.conv2d(input, weight, self.bias, self.stride,
--> 420                         self.padding, self.dilation, self.groups)
    421
    422     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

If anyone could help that would be so so so appreciated! I've been really stuck despite searching all over-- casting my UNet model to floats has not helped and I think I've called .cuda() where I'm supposed to.
Specific things I've tried:

RuntimeError: Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same loading entries of my DataLoader to CUDA
adding pin_memory to my DataLoader
loading my skorch NeuralNetwork to CUDA as stated here Pytorch, INPUT (normal tensor) and WEIGHT (cuda tensor) mismatch (which didn't work because it's not a function in skorch)
Casting my data to float (https://discuss.pytorch.org/t/input-type-torch-cuda-doubletensor-and-weight-type-torch-cuda-floattensor-should-be-the-same/22704)

Output:
cv2.imread gives you np.uint8 data type which will be converted to PyTorch's byte. The byte type cannot be used with the float type (which is most probably used by your model).
You need to convert the byte type to float type (and to Tensor), by modifying the dataset
import torchvision.transforms as transforms
class ImagesDataset(Dataset):
    """Constructs dataset of satellite images + masks"""
    def __init__(self, image_paths):
        super().__init__()
        self.image_paths = image_paths
        self.transform = transforms.Compose([transforms.ToTensor()])

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        print("idx:", idx)
        sample_dir = self.image_paths[idx]
        img_path = sample_dir +"/images/"+ Path(sample_dir).name +'.png'
        mask_path = sample_dir +'/mask.png'
        img, mask = cv2.imread(img_path), cv2.imread(mask_path)
        img = self.transform(img)
        mask = self.transform(mask)
        print("shape of img", img.shape)
        return img, mask


Instruction: How to concatenate 2 pytorch models and make the first one non-trainable in PyTorch

Input:
I've two networks, which I need to concatenate for my full model. However my first model is pre-trained and I need to make it non-trainable when training the full model. How can I achieve this in PyTorch.
I am able to concatenate two models using this answer
class MyModelA(nn.Module):
    def __init__(self):
        super(MyModelA, self).__init__()
        self.fc1 = nn.Linear(10, 2)

    def forward(self, x):
        x = self.fc1(x)
        return x

class MyModelB(nn.Module):
    def __init__(self):
        super(MyModelB, self).__init__()
        self.fc1 = nn.Linear(20, 2)

    def forward(self, x):
        x = self.fc1(x)
        return x

class MyEnsemble(nn.Module):
    def __init__(self, modelA, modelB):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB

    def forward(self, x):
        x1 = self.modelA(x)
        x2 = self.modelB(x1)
        return x2

# Create models and load state_dicts
modelA = MyModelA()
modelB = MyModelB()
# Load state dicts
modelA.load_state_dict(torch.load(PATH))

model = MyEnsemble(modelA, modelB)
x = torch.randn(1, 10)
output = model(x)

Basically here, I want to load pre-trained modelA and make it non-trainable when training the Ensemble model.

Output:
You can freeze all parameters of the model you dont want to train, by setting requires_grad to false.
Like this:
for param in model.parameters():
    param.requires_grad = False

This should work for you.
Another way is to handle this in your train-loop:
modelA = MyModelA()
modelB = MyModelB()

criterionB = nn.MSELoss()
optimizerB = torch.optim.Adam(modelB.parameters(), lr=0.001)

for epoch in range(epochs):
    for samples, targets in dataloader:
        optimizerB.zero_grad()

        x = modelA.train()(samples)
        predictions = modelB.train()(samples)

        loss = criterionB(predictions, targets)
        loss.backward()
        optimizerB.step()

So you pass the output of modelA to modelB but you optimize just modelB.


Instruction: PyTorch gives incorrect results due to broadcasting

Input:
I want to run some neural net experiments with PyTorch, but a minimal test case is giving wrong answers. The test case sets up a simple neural network with two input variables and an output variable that is just the sum of the inputs, and tries learning it as a regression problem; I expect it to converge on zero mean squared error, but it actually converges on 0.165. It's probably because of the issue alluded to in the warning message; how can I fix it?
Code:
import torch
import torch.nn as nn

# data
Xs = []
ys = []
n = 10
for i in range(n):
    i1 = i / n
    for j in range(n):
        j1 = j / n
        Xs.append([i1, j1])
        ys.append(i1 + j1)

# torch tensors
X_tensor = torch.tensor(Xs)
y_tensor = torch.tensor(ys)

# hyperparameters
in_features = len(Xs[0])
hidden_size = 100
out_features = 1
epochs = 500

# model
class Net(nn.Module):
    def __init__(self, hidden_size):
        super(Net, self).__init__()
        self.L0 = nn.Linear(in_features, hidden_size)
        self.N0 = nn.ReLU()
        self.L1 = nn.Linear(hidden_size, 1)

    def forward(self, x):
        x = self.L0(x)
        x = self.N0(x)
        x = self.L1(x)
        return x

model = Net(hidden_size)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.1)

# train
print("training")
for epoch in range(1, epochs + 1):
    # forward
    output = model(X_tensor)
    cost = criterion(output, y_tensor)

    # backward
    optimizer.zero_grad()
    cost.backward()
    optimizer.step()

    # print progress
    if epoch % (epochs // 10) == 0:
        print(f"{epoch:6d} {cost.item():10f}")
print()

output = model(X_tensor)
cost = criterion(output, y_tensor)
print("mean squared error:", cost.item())

Output:
training
C:\Users\russe\Anaconda3\envs\torch2\lib\site-packages\torch\nn\modules\loss.py:445: UserWarning: Using a target size (torch.Size([100])) that is different to the input size (torch.Size([100, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
    50   0.167574
   100   0.165108
   150   0.165070
   200   0.165052
   250   0.165039
   300   0.165028
   350   0.165020
   400   0.165013
   450   0.165009
   500   0.165006

mean squared error: 0.1650056540966034

And the message:

UserWarning: Using a target size (torch.Size([100])) that is different to the input size (torch.Size([100, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.

Output:
You're going to be a bit more specific on which tensors (X, or Y), but we can can reshape our tensors by using the torch.view function.
For example:
Y_tensor = torch.tensor(Ys)
print(Y_tensor.shape)
>> torch.Size([5])

new_shape = (len(Ys), 1)
Y_tensor = Y_tensor.view(new_shape)
print(Y_tensor.shape)
>> torch.Size([5, 1])

However, I'm skeptical that this broadcasting behavior is why you're having accuracy issues.


Instruction: RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed pytorch

Input:
How do I clear the gradient before calling the .backward() for the second time.

RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

a = torch.tensor([2.0], requires_grad = True)
b = torch.tensor([2.0], requires_grad = True)
d = torch.tensor([2.0], requires_grad = True)
c=a*b
c.backward()
e = d*e
e.backward(retain_graph=True)

I tried doing this: c.zero_grad() but I got the error c has no method zero_grad()

Output:
As the error message reads, you need to specify the retain_graph=True option on the first .backward call, not the second:
c.backward(retain_graph=True)
e = d*c
e.backward()

If you do not retain the graph, the second backward pass will not be able to reach the nodes c, a, and b because the activations will have been cleared by the first backward pass.


Instruction: What do the logits and probabilities from RobertaForSequenceClassification represent?

Input:
Being new to the "Natural Language Processing" scene, I am experimentally learning and have implemented the following segment of code:
from transformers import RobertaTokenizer, RobertaForSequenceClassification
import torch

path = "D:/LM/rb/"
tokenizer = RobertaTokenizer.from_pretrained(path)
model = RobertaForSequenceClassification.from_pretrained(path)

inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
outputs = model(**inputs)
pred_logits = outputs.logits
print(pred_logits)
probs = pred_logits.softmax(dim=-1).detach().cpu().flatten().numpy().tolist()
print(probs)

I understand that applying the model returns a "torch.FloatTensor comprising various elements depending on the configuration (RobertaConfig) and inputs", and that the logits are accessible using .logits. As demonstrated I have applied the .softmax function to the tensor to return normalised probabilities and have converted the result into a list. I am outputted with the following:
[0.5022980570793152, 0.49770188331604004]

Do these probabilities represent some kind of overall "masked" probability?
What do the first and second index represent in context of the input?

EDIT:
model.num_labels

Output:
2

@cronoik explains that the model "tries to classify if a sequence belongs to one class or another"
Am I to assume that because there are no trained output layers these classes don't mean anything yet?
For example, I can assume that the probability that the sentence, post analysis, belongs to class 1 is 0.5. However, what is class 1?
Additionally, model cards with pre-trained output layers such as the open-ai detector help differentiate between what is "real" and "fake", and so I can assume the class that a sentence belongs to. However, how can I confirm these "labels" without some type of "mapping.txt" file?

Output:
You have initialized a RobertaForSequenceClassification model that per default (in case of roberta-base and roberta-large which have no trained output layers for sequence classification) tries to classify if a sequence belongs to one class or another. I used the expression "belongs to one class or another" because these classes have no meaning yet. The output layer is untrained and it requires a finetuning to give these classes a meaning. Class 0 could be X and Class 1 could be Y or the other way around. For example, the tutorial for finetuning a sequence classification model for the IMDb review dataset defines negative reviews as Class 0 and positive reviews as Class 1 (link).
You can check the number of supported classes with:
model.num_labels

Output:
2

The output you get is the non-normalized probability for each class (i.e. logits). You applied the softmax function to normalize these probabilities, which leads to 0.5022980570793152 for the first class and 0.49770188331604004 for the second class.
Maybe you got confused because the values are close to each other. Let's try a model with a pretrained output layer (model card):
sentimodel = RobertaForSequenceClassification.from_pretrained('cardiffnlp/twitter-roberta-base-sentiment')
print(sentimodel.num_labels)
outputs = sentimodel(**inputs)
print(outputs.logits.softmax(dim=-1).tolist())

Output:
3
[[0.0015561950858682394, 0.019568447023630142, 0.9788752794265747]]

These values represent the probabilities for the sentence Hello, my dog is cute to be negative, neutral, or positive. We know what these classes are because the authors provided mapping that clarifies it. In case the authors of the model do not provide such a mapping (via a readme or the original training code), we can only guess what each class represents by testing it with random samples.
The model card you have mentioned does not provide any useful information regarding the mapping of the classes to what they represent, but the model is provided by huggingface itself and they provide a link to the code used for training the model. The dataset.py indicates that fake is represented by Class 0 and real by Class 1.


Instruction: The conflict is caused by: The user requested tensorboard==2.1.0 tensorflow 1.15.4 depends on tensorboard=1.15.0

Input:
I am trying to install a package VIBE from a git repo and inistally I was installing its dependencies. The code is located here: https://github.com/mkocabas/VIBE how should I fix this?
Here's the error I got:
(vibe-env) mona@mona:~/research/VIBE$ pip install -r requirements.txt
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Requirement already satisfied: torchvision==0.5.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 19)) (0.5.0)
Collecting git+https://github.com/mattloper/chumpy.git (from -r requirements.txt (line 24))
  Cloning https://github.com/mattloper/chumpy.git to /tmp/pip-req-build-vdh2h3jw
Collecting git+https://github.com/mkocabas/yolov3-pytorch.git (from -r requirements.txt (line 25))
  Cloning https://github.com/mkocabas/yolov3-pytorch.git to /tmp/pip-req-build-ay_gkil2
Collecting git+https://github.com/mkocabas/multi-person-tracker.git (from -r requirements.txt (line 26))
  Cloning https://github.com/mkocabas/multi-person-tracker.git to /tmp/pip-req-build-l9jgk1qb
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Collecting filterpy==1.4.5
  Using cached filterpy-1.4.5-py3-none-any.whl
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting gdown==3.6.4
  Downloading gdown-3.6.4.tar.gz (5.2 kB)
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Collecting h5py==2.10.0
  Using cached h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting joblib==0.14.1
  Downloading joblib-0.14.1-py2.py3-none-any.whl (294 kB)
     |████████████████████████████████| 294 kB 5.6 MB/s
Collecting llvmlite==0.32.1
  Downloading llvmlite-0.32.1-cp37-cp37m-manylinux1_x86_64.whl (20.2 MB)
     |████████████████████████████████| 20.2 MB 14.1 MB/s
Collecting matplotlib==3.1.3
  Using cached matplotlib-3.1.3-cp37-cp37m-manylinux1_x86_64.whl (13.1 MB)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting numba==0.47.0
  Downloading numba-0.47.0-cp37-cp37m-manylinux1_x86_64.whl (3.7 MB)
     |████████████████████████████████| 3.7 MB 33.0 MB/s
Requirement already satisfied: setuptools in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from numba==0.47.0->-r requirements.txt (line 6)) (51.0.0.post20201207)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting opencv-python==4.1.2.30
  Downloading opencv_python-4.1.2.30-cp37-cp37m-manylinux1_x86_64.whl (28.3 MB)
     |████████████████████████████████| 28.3 MB 29.4 MB/s
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting pillow==6.2.1
  Downloading Pillow-6.2.1-cp37-cp37m-manylinux1_x86_64.whl (2.1 MB)
     |████████████████████████████████| 2.1 MB 107.9 MB/s
Collecting progress==1.5
  Downloading progress-1.5.tar.gz (5.8 kB)
Collecting pyrender==0.1.36
  Downloading pyrender-0.1.36-py3-none-any.whl (1.2 MB)
     |████████████████████████████████| 1.2 MB 23.0 MB/s
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Collecting PyYAML==5.3.1
  Using cached PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl
Collecting scikit-image==0.16.2
  Downloading scikit_image-0.16.2-cp37-cp37m-manylinux1_x86_64.whl (26.5 MB)
     |████████████████████████████████| 26.5 MB 25.7 MB/s
Collecting scikit-video==1.1.11
  Using cached scikit_video-1.1.11-py2.py3-none-any.whl (2.3 MB)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting scipy==1.4.1
  Using cached scipy-1.4.1-cp37-cp37m-manylinux1_x86_64.whl (26.1 MB)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting smplx==0.1.13
  Downloading smplx-0.1.13-py3-none-any.whl (26 kB)
Requirement already satisfied: torch>=1.0.1.post2 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from smplx==0.1.13->-r requirements.txt (line 7)) (1.4.0)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Collecting tensorboard==2.1.0
  Downloading tensorboard-2.1.0-py3-none-any.whl (3.8 MB)
     |████████████████████████████████| 3.8 MB 29.3 MB/s
Requirement already satisfied: setuptools in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from numba==0.47.0->-r requirements.txt (line 6)) (51.0.0.post20201207)
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Requirement already satisfied: wheel>=0.26 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from tensorboard==2.1.0->-r requirements.txt (line 18)) (0.36.1)
Collecting tensorflow==1.15.4
  Downloading tensorflow-1.15.4-cp37-cp37m-manylinux2010_x86_64.whl (110.5 MB)
     |████████████████████████████████| 110.5 MB 22 kB/s
Requirement already satisfied: numpy==1.17.5 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.17.5)
Requirement already satisfied: six>=1.11.0 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from chumpy==0.70->-r requirements.txt (line 24)) (1.15.0)
Requirement already satisfied: wheel>=0.26 in /home/mona/anaconda3/envs/vibe-env/lib/python3.7/site-packages (from tensorboard==2.1.0->-r requirements.txt (line 18)) (0.36.1)
INFO: pip is looking at multiple versions of tensorboard to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of smplx to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of scipy to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of scikit-video to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of scikit-image to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of pyyaml to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of pyrender to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of progress to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of pillow to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of opencv-python to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of numpy to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of numba to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of multi-person-tracker to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of matplotlib to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of llvmlite to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of joblib to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of h5py to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of gdown to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of filterpy to determine which version is compatible with other requirements. This could take a while.
INFO: pip is looking at multiple versions of chumpy to determine which version is compatible with other requirements. This could take a while.
ERROR: Cannot install -r requirements.txt (line 17) and tensorboard==2.1.0 because these package versions have conflicting dependencies.

The conflict is caused by:
    The user requested tensorboard==2.1.0
    tensorflow 1.15.4 depends on tensorboard<1.16.0 and >=1.15.0

To fix this you could try to:
1. loosen the range of package versions you've specified
2. remove package versions to allow pip attempt to solve the dependency conflict

ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/user_guide/#fixing-conflicting-dependencies

(vibe-env) mona@mona:~/research/VIBE$ python
Python 3.7.9 (default, Aug 31 2020, 12:42:55)
[GCC 7.3.0] :: Anaconda, Inc. on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.__version__
'1.4.0'

Here are all the commands I ran before this:
(base) mona@mona:~/research/VIBE$ export CONDA_ENV_NAME=vibe-env

(base) mona@mona:~/research/VIBE$ conda create -n $CONDA_ENV_NAME python=3.7

(base) mona@mona:~/research/VIBE$ eval "$(conda shell.bash hook)"

(base) mona@mona:~/research/VIBE$ conda activate $CONDA_ENV_NAME

(vibe-env) mona@mona:~/research/VIBE$ pip install numpy==1.17.5 torch==1.4.0 torchvision==0.5.0

(vibe-env) mona@mona:~/research/VIBE$ pip install git+https://github.com/giacaglia/pytube.git --upgrade

Output:
The key here is this:
The conflict is caused by:
    The user requested tensorboard==2.1.0
    tensorflow 1.15.4 depends on tensorboard<1.16.0 and >=1.15.0

This is due to the fact that there is a conflict in requirements.txt of https://github.com/mkocabas/VIBE since it requires tensorboard==2.1.0 and tensorflow==1.15.4. However, according to the error message, this version of tensorflow only works with tensorboard 1.15.0 - 1.15.x.
If you read the error closely you will see that pip itself suggests how to resolve this:

To fix this you could try to:

loosen the range of package versions you've specified
remove package versions to allow pip attempt to solve the dependency conflict


Instruction: Custom PyTorch optimizer is not working properly and am unable to access gradients

Input:
I'm trying to implement the elastic averaging stochastic gradient descent (EASGD) algorithm from the paper Deep Learning with Elastic Averaging SGD and was running into some trouble.
I'm using PyTorch's torch.optim.Optimizer class and referencing the official implementation of SGD and the official implementation of Accelerated SGD in order to start off somewhere.
The code that I have is:
import torch.optim as optim

class EASGD(optim.Optimizer):
    def __init__(self, params, lr, tau, alpha=0.001):
        self.alpha = alpha

        if lr < 0.0:
            raise ValueError(f"Invalid learning rate {lr}.")

        defaults = dict(lr=lr, alpha=alpha, tau=tau)
        super(EASGD, self).__init__(params, defaults)

    def __setstate__(self, state):
        super(EASGD, self).__setstate__(state)

    def step(self, closure=None):
        loss = None
        if closure is not None:
            with torch.enable_grad():
                loss = closure()

        for group in self.param_groups:
            tau = group['tau']

            for t, p in enumerate(group['params']):
                x_normal = p.clone()
                x_tilde = p.clone()

                if p.grad is None:
                    continue

                if t % tau == 0:
                    p = p - self.alpha * (x_normal - x_tilde)
                    x_tilde = x_tilde + self.alpha * (x_normal - x_tilde)

                d_p = p.grad.data
                p.data.add_(d_p, alpha=-group['lr'])

        return loss

When I run this code, I get the following error:

/home/user/github/test-repo/easgd.py:50: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.

Reading this PyTorch Discussion helped understand what the difference between leaf and non-leaf variables are, but I'm not sure how I should fix my code to make it work properly.
Any tips on what to do or where to look are appreciated. Thanks.

Output:
I think the issue is you are copying p on this line:
p = p - self.alpha * (x_normal - x_tilde)

If this line gets execute (which the case in the first cycle when t=0) the following line raises the error because p doesn't have .grad attribute anymore.
You should use inplace operators instead, add_, mult_, sub_, divide_, etc...
for t, p in enumerate(group['params']):
    if p.grad is None:
        continue
    d_p = p.grad.data

    if t % tau == 0:
        d_p.sub_(self.alpha*0.01)

    p.data.add_(d_p, alpha=-group['lr'])

Above, I have removed x_normal, x_tilde since you didn't give them proper values. But I hope you get the idea. Only use inplace operator when working with the data inside the step function.


Instruction: How to dynamically update batch norm momentum in TF2?

Input:
I found a PyTorch implementation that decays the batch norm momentum parameter from 0.1 in the first epoch to 0.001 in the final epoch. Any suggestions on how to do this with the batch norm momentum parameter in TF2? (i.e., start at 0.9 and end at 0.999) For example, this is what is done in the PyTorch code:
# in training script
momentum = initial_momentum * np.exp(-epoch/args.epochs * np.log(initial_momentum/final_momentum))
model_pos_train.set_bn_momentum(momentum)

# model class function
def set_bn_momentum(self, momentum):
    self.expand_bn.momentum = momentum
    for bn in self.layers_bn:
        bn.momentum = momentum

SOLUTION:
The selected answer below provides a viable solution when using the tf.keras.Model.fit() API. However, I was using a custom training loop. Here is what I did instead:
After each epoch:
mi = 1 - initial_momentum  # i.e., inital_momentum = 0.9, mi = 0.1
mf = 1 - final_momentum  # i.e., final_momentum = 0.999, mf = 0.001
momentum = 1 - mi * np.exp(-epoch / epochs * np.log(mi / mf))
model = set_bn_momentum(model, momentum)

set_bn_momentum function (credit to this article):
def set_bn_momentum(model, momentum):
    for layer in model.layers:
        if hasattr(layer, 'momentum'):
            print(layer.name, layer.momentum)
            setattr(layer, 'momentum', momentum)

    # When we change the layers attributes, the change only happens in the model config file
    model_json = model.to_json()

    # Save the weights before reloading the model.
    tmp_weights_path = os.path.join(tempfile.gettempdir(), 'tmp_weights.h5')
    model.save_weights(tmp_weights_path)

    # load the model from the config
    model = tf.keras.models.model_from_json(model_json)

    # Reload the model weights
    model.load_weights(tmp_weights_path, by_name=True)
    return model

This method did not add significant overhead to the training routine.

Output:
You can set an action in the begin/the end of each batch, so you can control the any parameter during the epoch.
Below the options for the callbacks:
class CustomCallback(keras.callbacks.Callback):
    def on_epoch_begin(self, epoch, logs=None):
        keys = list(logs.keys())
        print("Start epoch {} of training; got log keys: {}".format(epoch, keys))

    def on_epoch_end(self, epoch, logs=None):
        keys = list(logs.keys())
        print("End epoch {} of training; got log keys: {}".format(epoch, keys))

    def on_train_batch_begin(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Training: start of batch {}; got log keys: {}".format(batch, keys))

    def on_train_batch_end(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Training: end of batch {}; got log keys: {}".format(batch, keys))

    def on_test_batch_begin(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Evaluating: start of batch {}; got log keys: {}".format(batch, keys))

    def on_test_batch_end(self, batch, logs=None):
        keys = list(logs.keys())
        print("...Evaluating: end of batch {}; got log keys: {}".format(batch, keys))

You can access the momentum
batch = tf.keras.layers.BatchNormalization()
batch.momentum = 0.001

Inside the model you have to specified the correct layer
model.layers[1].momentum = 0.001

You can find more information and example at writing_your_own_callbacks


Instruction: What is the numpy equivalent of expand in pytorch?

Input:
Suppose I have a numpy array x of shape [1,5]. I want to expand it along axis 0 such that the resulting array y has shape [10,5] and y[i:i+1,:] is equal to x for each i.
If x were a pytorch tensor I could simply do
y = x.expand(10,-1)

But there is no expand in numpy and the ones that look like it (expand_dims and repeat) don't seem to behave like it.

Example:
>>> import torch
>>> x = torch.randn(1,5)
>>> print(x)
tensor([[ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724]])
>>> print(x.expand(10,-1))
tensor([[ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724],
        [ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724]])

Output:
You can achieve that with np.broadcast_to. But you can't use negative numbers:
>>> import numpy as np
>>> x = np.array([[ 1.3306,  0.0627,  0.5585, -1.3128, -1.4724]])
>>> print(np.broadcast_to(x,(10,5)))
[[ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]
 [ 1.3306  0.0627  0.5585 -1.3128 -1.4724]]


Instruction: Does model.train() put every thing in train mode in pytorch even sub-networks?

Input:
I wrote a neural network in PyTorch, which uses ResNet as feature generator, and it is fined tuned with the whole network.
My model consists of Resnet and several layers that I added to it. My question is:
When I call model.train(), Does it put ResNet in train mode or I should call train on it separately?

Output: By the look of it, if you call train() on a module, it will call train() recursively on all children. So model.train() - model being the model containing the Resnet - will suffice.


Instruction: Multiple values for argument

Input:
I am trying to convert this code  passing it with pysyft refference
like this :
class SyNet(sy.Module):
  def __init__(self,embedding_size, num_numerical_cols, output_size, layers, p ,torch_ref):
    super(SyNet, self ).__init__(  embedding_size, num_numerical_cols , output_size , layers , p=0.4  ,torch_ref=torch_ref  )
    self.all_embeddings=self.torch_ref.nn.ModuleList([nn.Embedding(ni, nf) for ni, nf in embedding_size])
    self.embedding_dropout=self.torch_ref.nn.Dropout(p)
    self.batch_norm_num=self.torch_ref.nn.BatchNorm1d(num_numerical_cols)

    all_layers= []
    num_categorical_cols = sum((nf for ni, nf in embedding_size))
    input_size = num_categorical_cols + num_numerical_cols

    for i in layers:
      all_layers.append(self.torch_ref.nn.Linear(input_size,i))
      all_layers.append(self.torch_ref.nn.ReLU(inplace=True))
      all_layers.append(self.torch_ref.nn.BatchNorm1d(i))
      all_layers.append(self.torch_ref.nn.Dropout(p))
      input_size = i

    all_layers.append(self.torch_ref.nn.Linear(layers[-1], output_size))

    self.layers = self.torch_ref.nn.Sequential(*all_layers)

  def forward(self, x_categorical, x_numerical):
    embeddings= []
    for i,e in enumerate(self.all_embeddings):
      embeddings.append(e(x_categorical[:,i]))

    x_numerical = self.batch_norm_num(x_numerical)
    x = self.torch_ref.cat([x, x_numerical], 1)
    x = self.layers(x)
    return x

But when I try to create a instance of the model
model = SyNet( categorical_embedding_sizes, numerical_data.shape[1], 2, [200,100,50], p=0.4 ,torch_ref= th)

I got a TypeError
TypeError: multiple values for argument 'torch_ref'
I tried to change the order of the arguments but i got an error about positional arguments .
Can you help me , I am not very experienced in classes and functions (oop)
Thank you in advance !

Output:
Looking at PySyft source code for Module. The constructor of your class parent only takes a single argument: torch_ref.
You should therefore call the super constructor with:
super(SyNet, self).__init__(torch_ref=torch_ref) # line 3

removing all arguments but torch_ref from the call.


Instruction: Python: fastest way to check in which interval number is in

Input:
Suppose I have split the interval [0, 1] into a series of smaller intervals [0, 0.2), [0.2, 0.4), [0.4, 0.9), [0.9, 1.0]. Now I sample a value r in [0, 1]. What is the fastest way I can check in which interval this belongs to using Python / Numpy / Pytorch?
The obvious way is this:
r = np.random.rand()
if 0 <= r < 0.2:
    pass # do something
elif 0.2 <= r < 0.4:
    pass # do something else
elif 0.4 <= r < 0.9:
    pass # do yet something else again
elif 0.9 <= r <= 1.0:
    pass # do some other thing

Output:
You'll want to first transform your list of intervals into a list of boundaries, so instead of many intervals [0, 0.2), [0.2, 0.4), [0.4, 0.9), [0.9, 1.0], you just define:
boundaries = [0, 0.2, 0.4, 0.9, 1.0]  # values must be sorted!!

Then you can perform a binary search over all of them, to see in which segment a value belongs:
index = bisect.bisect_right(boundaries, value)

index will be the index of the upper bound, so to get the range, you'd to:
range_low = boundaries[index - 1] if index > 0 else None
range_high = boundaries[index] if index < len(boundaries) else None

This will also take care of handling values which are not in any of the intervals. The binary search will be done in log(N) compares, which is the theoretical best thing you can do for arbitrary intervals.


Instruction: Pytorch 1.7.0 | DataLoader Error - TypeError: 'module' object is not callable

Input:
This is my code, I am using pycharm!
Imports
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.utils.data as DataLoader
import torchvision.datasets as Datasets
import torchvision.transforms as transforms

create Fully Connected Network
class NN(nn.Module):
    def __init__(self, input_size, num_classes): #(28x28 = 784)
        super(NN, self).__init__()
        self.fc1 = nn.Linear(input_size, 50)
        self.fc2 = nn.Linear(50, input_size) #hidden layer

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x

#Set Device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

Hyperparameters
input_size = 784
num_classes = 10
learning_rate = 0.001
batch_size = 2
num_epochs = 1

Load Data
train_dataset = Datasets.MNIST(root='dataset/', train=True, transform=transforms.ToTensor(), download=True)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_dataset = Datasets.MNIST(root='dataset/', train=False, transform=transforms.ToTensor(), download=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)

Initialize network
model = NN(input_size=input_size, num_classes=num_classes).to(device)

Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

Train network
for epoch in range(num_epochs):
    for batch_idx, (data, targets) in enumerate(train_loader):
        data = data.to(device=device)
        targets = targets.to(device=device)

        print(data.shape)

I am getting the error on this line
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
Error is
in <module>
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
TypeError: 'module' object is not callable

Output:
You need to edit your import
from torch.utils.data import DataLoader, Dataset


Instruction: Is it safe to truncate torchaudio's loaded 16-bit audios to `float16` from `float32`?

Input:
I have multiple WAV files with 16 bits of depth/precision. torchaudio.info(...) recognizes this, giving me:
precision = {int} 16

Yet when I use torchaudio.load(...), I get a float32 dtype for the resulting tensor.
With a tensor called audio, I know that I can do audio.half() to truncate it to 16 bits, reducing memory usage of my dataset. But is this an operation that will preserve precision of all possible original values? I'm not lowering the dtype's precision below the original audio's precision, but there may be a good reason I'm unaware of as to why torchaudio still returns float32.

Output:
I would say it's returned as float32 because this is pytorch's default datatype. So if you create any models with weights, they'll be float32 as well. Therefore, the inputs will be incompatible with the model if you make the conversion on the input data. (E: or it will silently convert your data to 32 bit anyway, to make it compatible with your model. Not sure which pytorch opt for, but tensorflow definitely throws the error).
Look at setting the default datatype to float16, before creating any models, if you're looking to make small models: https://pytorch.org/docs/stable/generated/torch.set_default_dtype.html
HOWEVER note that you will lose 5 bits of precision if you convert a 16-bit int, as you have diagnosed that the number actually is (but is represented as a 32-bit float), to a 16-bit float. This is because 5 bits of precision are used in the exponent, leaving just 10 bits to represent the decimal part representing the number.
I would just keep it at float32, if you're not particularly memory constrained.


Instruction: Metrics mismatch between BertForSequenceClassification Class and my custom Bert Classification

Input:
I implemented my custom Bert Binary Classification Model class, by adding a classifier layer on top of Bert Model (attached below). However, the accuracy/metrics are significantly different when I train with the official BertForSequenceClassification model, which makes me wonder if I am missing somehting in my class.
Few Doubts I have:
While loading the official BertForSequenceClassification from_pretrained are the classifiers weight initialized as well from pretrained model or they are randomly initialized? Because in my custom class they are randomly initialized.
class MyCustomBertClassification(nn.Module):
    def __init__(self, encoder='bert-base-uncased',
                        num_labels,
                        hidden_dropout_prob):

    super(MyCustomBertClassification, self).__init__()
    self.config  = AutoConfig.from_pretrained(encoder)
    self.encoder = AutoModel.from_config(self.config)
    self.dropout = nn.Dropout(hidden_dropout_prob)
    self.classifier = nn.Linear(self.config.hidden_size, num_labels)

def forward(self, input_sent):
    outputs = self.encoder(input_ids=input_sent['input_ids'],
                          attention_mask=input_sent['attention_mask'],
                          token_type_ids=input_sent['token_type_ids'],
                          return_dict=True)

    pooled_output = self.dropout(outputs[1])
    # for both tasks
    logits = self.classifier(pooled_output)

    return logits

Output:
Each model tells you via a warning message which layers are randomly initialized when you use the method from_pretrained:
from transformers import BertForSequenceClassification

b = BertForSequenceClassification.from_pretrained('bert-base-uncased')

Output:
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']

The difference between your implementation and the BertForSequenceClassification is that you do not use any pretrained weights at all. The method from_config does not load the pretrained weights from a state_dict:
import torch
from transformers import AutoModelForSequenceClassification, AutoConfig

b2 = AutoModelForSequenceClassification.from_config(AutoConfig.from_pretrained('bert-base-uncased'))
b3 = AutoModelForSequenceClassification.from_pretrained('bert-base-uncased')

print("Does from_config provides pretrained weights: {}".format(torch.equal(b.bert.embeddings.word_embeddings.weight, b2.base_model.embeddings.word_embeddings.weight)))
print("Does from_pretrained provides pretrained weights: {}".format(torch.equal(b.bert.embeddings.word_embeddings.weight, b3.base_model.embeddings.word_embeddings.weight)))

Output:
Does from_config provides pretrained weights: False
Does from_pretrained provides pretrained weights: True

Therefore you probably want to change your class to:
class MyCustomBertClassification(nn.Module):
    def __init__(self, encoder='bert-base-uncased',
                       num_labels=2,
                       hidden_dropout_prob=0.1):

       super(MyCustomBertClassification, self).__init__()
       self.config  = AutoConfig.from_pretrained(encoder)
       self.encoder = AutoModel.from_pretrained(encoder)
       self.dropout = nn.Dropout(hidden_dropout_prob)
       self.classifier = nn.Linear(self.config.hidden_size, num_labels)

    def forward(self, input_sent):
       outputs = self.encoder(input_ids=input_sent['input_ids'],
                         attention_mask=input_sent['attention_mask'],
                         token_type_ids=input_sent['token_type_ids'],
                         return_dict=True)

       pooled_output = self.dropout(outputs[1])
       # for both tasks
       logits = self.classifier(pooled_output)

       return logits

myB = MyCustomBertClassification()

print(torch.equal(b.bert.embeddings.word_embeddings.weight, myB.encoder.embeddings.word_embeddings.weight))

Output:
True


Instruction: How to get the second moment of the gradient

Input:
In the OpenAI Five paper it is mentioned that the "Gradients are additionally clipped per parameter to be within between ±5√v where v is the running estimate of the second moment of the (unclipped) gradient.". This is something I would like to implement in my project, but I am not sure how to do it neihter in theory nor in practice.
From wikipedia I found out that the "The second central moment is the variance. The positive square root of the variance is the standard deviation [...]". My best guess regarding the "running estimate" is that it is the Exponential Moving Average. The gradients of a network can be accessed as this comment suggests.
From these I would assume that √v is the Exponential Running Average of the standard dev. of the gradients and could be calculated via:
estimate = alpha * torch.std(list(param.grad for param in model.parameters())) + (1-alpha) * estimate
Is my theory correct? Is there a better way to do it?
Thanks in advance.
Edit: fixed gradient gathering after Mr. For Example"s answer.

Output:
I think you are on the right path, my guess is basically as same as yours, just with little different.
First, what is moment?
N-th moment of a random variable is defined as the expected value of that variable to the power of n. More formally:

m — moment, X — random variable
So the first moment is mean, and the second moment is uncentered variance (meaning we don’t subtract the mean during variance calculation), intuitively, clipping the gradients by moving average of its standard deviation wrt zero make sense.
Second, what is the correct code?
list(network.parameters()) only give you the parameters, for get gradient of each parameters you need [param.grad for param in network.parameters()]
Given all the things we know above, the correct code should be (you can try to optimize it by all means):
grads_square = torch.FloatTensor([torch.square(param.grad) for param in network.parameters()])
estimate = alpha * torch.sqrt(torch.mean(grads_square)) + (1-alpha) * estimate


Instruction: What is the Problem in my Building Softmax from Scratch in Pytorch

Input:
I read this post ans try to build softmax by myself. Here is the code
import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import time
import sys
import numpy as np

#============================ get the dataset =========================

mnist_train = torchvision.datasets.FashionMNIST(root='~/Datasets/FashionMNIST', train=True, download=True, transform=transforms.ToTensor())
mnist_test = torchvision.datasets.FashionMNIST(root='~/Datasets/FashionMNIST', train=False, download=True, transform=transforms.ToTensor())

batch_size = 256
num_workers = 0

train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)

#============================     train      =========================
num_inputs = 28 * 28
num_outputs = 10
epochs = 5
lr = 0.05

# Initi the Weight and bia
W = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_outputs)), dtype=torch.float)
b = torch.zeros(num_outputs, dtype=torch.float)
W.requires_grad_(requires_grad = True)
b.requires_grad_(requires_grad=True)

# softmax function
def softmax(X):
    X = X.exp()
    den = X.sum(dim=1, keepdim=True)
    return X / den

# loss
def cross_entropy(y_hat, y):
    return - torch.log(y_hat.gather(1, y.view(-1, 1))).sum()

# accuracy function
def accuracy(y_hat, y):
    return (y_hat.argmax(dim=1) == y).float().mean().item()

for epoch in range(epochs):

    train_loss_sum = 0.0
    train_acc_sum = 0.0
    n_train = 0

    for X, y in train_iter:
        # X.shape: [256, 1, 28, 28]
        # y.shape: [256]

        # flatten the X into [256, 28*28]
        X = X.flatten(start_dim=1)
        y_pred = softmax(torch.mm(X, W) + b)

        loss = cross_entropy(y_pred, y)

        loss.backward()

        W.data = W.data - lr * W.grad
        b.data = b.data - lr* b.grad

        W.grad.zero_()
        b.grad.zero_()

        train_loss_sum += loss.item()

        train_acc_sum += accuracy(y_pred, y)
        n_train += y.shape[0]

    # evaluate the Test

    test_acc, n_test = 0.0, 0
    with torch.no_grad():

        for X_test, y_test in test_iter:
            X_test = X_test.flatten(start_dim=1)
            y_test_pred = softmax(torch.mm(X_test, W) + b)
            test_acc += accuracy(y_test_pred, y_test)
            n_test += y_test.shape[0]

    print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
              % (epoch + 1, train_loss_sum/n_train , train_acc_sum / n_train, test_acc / n_test))

Compare with original post, Here I turn
def cross_entropy(y_hat, y):
    return - torch.log(y_hat.gather(1, y.view(-1, 1)))

into
def cross_entropy(y_hat, y):
    return - torch.log(y_hat.gather(1, y.view(-1, 1))).sum()

Since the backward need a scalar.
However, My results are
epoch 1, loss nan, train acc 0.000, test acc 0.000
epoch 2, loss nan, train acc 0.000, test acc 0.000
epoch 3, loss nan, train acc 0.000, test acc 0.000
epoch 4, loss nan, train acc 0.000, test acc 0.000
epoch 5, loss nan, train acc 0.000, test acc 0.000

Any idea?
Thanks.

Output:
Change:
def cross_entropy(y_hat, y):
    return - torch.log(y_hat.gather(1, y.view(-1, 1))).sum()

To:
def cross_entropy(y_hat, y):
    return - torch.log(y_hat[range(len(y_hat)), y] + 1e-8).sum()

Outputs should be something like:
epoch 1, loss 9.2651, train acc 0.002, test acc 0.002
epoch 2, loss 7.8493, train acc 0.002, test acc 0.002
epoch 3, loss 6.6875, train acc 0.002, test acc 0.003
epoch 4, loss 6.0928, train acc 0.003, test acc 0.003
epoch 5, loss 5.1277, train acc 0.003, test acc 0.003

And be aware the problem of nan can also cause by X = X.exp() in the softmax(X), when X is too big then exp() will outputs inf, when this happen you could try to clip the X before using exp()


Instruction: How does max_length, padding and truncation arguments work in HuggingFace' BertTokenizerFast.from_pretrained('bert-base-uncased')?

Input:
I am working with Text Classification problem where I want to use the BERT model as the base followed by Dense layers. I want to know how does the 3 arguments work? For example, if I have 3 sentences as:
'My name is slim shade and I am an aspiring AI Engineer',
'I am an aspiring AI Engineer',
'My name is Slim'

SO what will these 3 arguments do? What I think is as follows:

max_length=5 will keep all the sentences as of length 5 strictly
padding=max_length will add a padding of 1 to the third sentence
truncate=True will truncate the first and second sentence so that their length will be strictly 5.

Please correct me if I am wrong.
Below is my code which I have used.
! pip install transformers==3.5.1

from transformers import BertTokenizerFast

tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')

tokens = tokenizer.batch_encode_plus(text,max_length=5,padding='max_length', truncation=True)

text_seq = torch.tensor(tokens['input_ids'])
text_mask = torch.tensor(tokens['attention_mask'])

Output:
What you have assumed is almost correct, however, there are few differences.
max_length=5, the max_length specifies the length of the tokenized text. By default, BERT performs word-piece tokenization. For example the word "playing" can be split into "play" and "##ing" (This may not be very precise, but just to help you understand about word-piece tokenization), followed by adding [CLS] token at the beginning of the sentence, and [SEP] token at the end of sentence. Thus, it first tokenizes the sentence, truncates it to max_length-2 (if truncation=True), then prepend [CLS] at the beginning and [SEP] token at the end.(So a total length of max_length)
padding='max_length', In this example it is not very evident that the 3rd example will be padded, as the length exceeds 5 after appending [CLS] and [SEP] tokens. However, if you have a max_length of 10. The tokenized text corresponds to [101, 2026, 2171, 2003, 11754, 102, 0, 0, 0, 0], where 101 is id of [CLS] and 102 is id of [SEP] tokens. Thus, padded by zeros to make all the text to the length of max_length
Likewise, truncate=True will ensure that the max_length is strictly adhered, i.e, longer sentences are truncated to max_length only if truncate=True


Instruction: Sorting a Pytorch Tensor by Trace

Input:
I have a (100,64,22,3,3) shaped pytorch tensor, and I would like to sort along axis=0 by the trace of the (3,3) components. The code I have below works, but it is very slow due to the for loops. Is there a way to vectorize the operation to speed it up?
x=torch.rand(100,64,22,3,3)
x_sorted=torch.zeros((x.shape[0],x.shape[1],x.shape[2],x.shape[3],x.shape[4]))
            for i in range(x.shape[0]):
              #compute tensorized trace
              trace=new=torch.diagonal(x[i], dim1=-2, dim2=-1).sum(-1)
              #Sort the trace
              trace_values,trace_ind=torch.sort(trace,dim=0,descending=True)
              for j in range(x_sorted.shape[1]):
                for k in range(x_sorted.shape[2]):
                  x_sorted[i,j,k]=x[i,trace_ind[j,k],k]

Output:
Try:
tensor = torch.tensor(np.random.rand(100,64, 3, 3))

orders = torch.argsort(torch.einsum('ijkk->ijk', tensor).sum(-1), axis=0)
orders.shape

tensor[orders, torch.arange(s.shape[1])[None, :]]


Instruction: Still confused about model.train()

Input:
I read all the posts here regarding model.train() and still didn't understand what is up with it. Specifically, when I use a pre-trained model like DenseNet or VGG with all parameters frozen beside the last layer not using drop-out nor Batch Normalization, the training loss starts off a lot smaller when using model.train(), but then decreases at about the same rate as when without it.
Why?

Output:
There are just three options: just model(inputs), model.train()(inputs) and model.eval()(inputs). The only difference is, that when using .eval() all the dropout and normalization is ignored because its just used for training and not for tesing.
Now you asked why it is still training when you just use model(inputs)? Because when you dont use train() nor eval() the model will be automatically in train-mode. So model(inputs) is the same as model.train()(inputs).


Instruction: How to create a neural network that takes in an image and ouputs another image?

Input:
I'm trying to create a neural network that has an image of the L (from Lab) format and output the ab dimensions. I am able to pass the L dimension without an issue, but I'm having trouble figuring out how to output the ab dimensions. The output should be for shape 1x2xHxW, where H and W are the height and width of the input image. Here is my network so far:
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        # Get the resnet18 model from torchvision.model library
        self.model = models.resnet18(pretrained=True)

        self.model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)

        # Replace fully connected layer of our model to a 2048 feature vector output
        self.model.classifier = nn.Sequential()

        # Add custom classifier layers
        self.fc1 = nn.Linear(1000, 1024)
        self.Dropout1 = nn.Dropout()
        self.PRelU1 = nn.PReLU()

        self.fc2 = nn.Linear(1024, 512)
        self.Dropout2 = nn.Dropout()
        self.PRelU2 = nn.PReLU()

        self.fc3 = nn.Linear(512, 256)
        self.Dropout3 = nn.Dropout()
        self.PRelU3 = nn.PReLU()

        self.fc4 = nn.Linear(256, 313)
        # self.PRelU3 = nn.PReLU()

        self.softmax = nn.Softmax(dim=1)
        self.model_out = nn.Conv2d(313, 2, kernel_size=1, padding=0, dilation=1, stride=1, bias=False)
        self.upsample4 = nn.Upsample(scale_factor=4, mode='bilinear')

    def forward(self, x):
        # x is our input data
        x = self.model(x)
        x = self.Dropout1(self.PRelU1(self.fc1(x)))
        x = self.Dropout2(self.PRelU2(self.fc2(x)))
        x = self.Dropout3(self.PRelU3(self.fc3(x)))
        x = self.softmax(self.fc4(x))

        return x

Output:
I dont really know what you mean by "ab dimensions" and Im not sure what the "L format" is but I can tell you how to use cnns to generate images.
Normally you would use an autoencoder but that depends on the task. An autoencoder takes a image as input and, similiar to a normal classification reduces the dimensions. But unlike in classification you dont flatten the featuremaps and add classification layers, but you unsample and deconvolve them. So first you "encode" the "image" and then you "decode" it. The middle layer before you start upsampling, is called the bottleneck. There are no dense layers and no softmax activations needed.
Here is an example how this would look like as a pytorch model (an Autoencoder for the cifar10 dataset):
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()

        """ encoder """
        self.conv1 = nn.Conv2d(3, 32, kernel_size=(5, 5))
        self.batchnorm1 = nn.BatchNorm2d(32)

        self.conv2 = nn.Conv2d(32, 64, kernel_size=(4, 4), stride=3)
        self.batchnorm2 = nn.BatchNorm2d(64)

        self.conv3 = nn.Conv2d(64, 128, kernel_size=(3, 3), stride=3)
        self.batchnorm3 = nn.BatchNorm2d(128)

        self.maxpool2x2 = nn.MaxPool2d(2)   # not in usage

        """ decoder """
        self.upsample2x2 = nn.Upsample(scale_factor=2)   # not in usage

        self.deconv1 = nn.ConvTranspose2d(128, 64, kernel_size=(3, 3), stride=3)
        self.batchnorm1 = nn.BatchNorm2d(64)

        self.deconv2 = nn.ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=3)
        self.batchnorm2 = nn.BatchNorm2d(32)

        self.deconv3 = nn.ConvTranspose2d(32, 3, kernel_size=(5, 5))
        self.batchnorm3 = nn.BatchNorm2d(3)

    def forward(self, x, train_: bool=True, print_: bool=False, return_bottlenecks: bool=False):

        """ encoder """
        x = self.conv1(x)
        x = self.batchnorm1(x)
        x = F.relu(x)

        x = self.conv2(x)
        x = self.batchnorm2(x)
        x = F.relu(x)

        x = self.conv3(x)
        x = self.batchnorm3(x)
        bottlenecks = F.relu(x)

        """ decoder """
        x = self.deconv1(bottlenecks)
        x = self.batchnorm1(x)
        x = F.relu(x)

        x = self.deconv2(x)
        x = self.batchnorm2(x)
        x = F.relu(x)

        x = self.deconv3(x)
        x = torch.sigmoid(x)

        return x

In this example I dont use "maxpool" and "upsample" but that depends on your model.
Upsample is basically the opposite of of maxpool, and you can see ConvTranspose2d also like the opposite of convolution (even though that wouldnt really be he right explanaition).
So you basically want the "decoder" part to be the opposite (or mirrored version) of the "encoder" part. Figuring out the dimensions kernel-sizes etc for each layer can be quite tricky but you basically have to set them so that the archtitecture is almost symmetrical and the output dimensions of the model are the size of the image you want to produce.
Thats what an "image producing" architecture could like like:

source: https://www.semanticscholar.org/paper/Feature-discovery-and-visualization-of-robot-data-Flaspohler-Roy/514a2f7461edd3e4c2d56d57f9002e1dc445eb58/figure/1


Instruction: Query padding mask and key padding mask in Transformer encoder

Input:
I'm implementing self-attention part in transformer encoder using pytorch nn.MultiheadAttention and confusing in the padding masking of transformer.
The following picture shows the self-attention weight of the query (row) and key (column).
As you can see, there are some tokens "<PAD>" and I have already mask it in key. Therefore the tokens will not calculate the attention weight.

There are still two questions:

In query part, can I also mask them("<PAD>") except for the red square part? Is this reasonable?

How can I mask "<PAD>" in the query?

The attention weights also use the softmax function along the row by giving mask in src_mask or src_key_padding_mask argument. If I set all the "<PAD>" row into -inf, the softmax will return nan and the loss with be nan

Output:
There is no need to mask the queries during self-attention, it should be enough if do not use the states corresponding to the <PAD> tokens later in the network (either as hidden states or keys/values), they will not influence the loss function nor anything else in the network.
If you want to make sure that you did not make a bug causing the gradient flowing through the <PAD> tokens you can explicitly zero-out the self-attention using torch.where after it is computed.


Instruction: How to convert sparse to dense adjacency matrix?

Input:
I am trying to convert a sparse adjacency matrix/list that only contains the indices of the non-zero elements ([[rows], [columns]]) to a dense matrix that contains 1s at the indices and otherwise 0s. I found a solution using to_dense_adj from Pytorch geometric (Documentation). But this does not exactly what I want, since the shape of the dense matrix is not as expected. Here is an example:
sparse_adj = torch.tensor([[0, 1, 2, 1, 0], [0, 1, 2, 3, 4]])

So the dense matrix should be of size 5x3 (the second array "stores" the columns; with non-zero elements at (0,0), (1,1), (2,2),(1,3) and (0,4)) because the elements in the first array are lower or equal than 2.
However,
dense_adj = to_dense(sparse_adj)[0]

outputs a dense matrix, but of shape (5,5). Is it possible to define the output shape or is there a different solution to get what I want?
Edit: I have a solution to convert it back to the sparse representation now that works
dense_adj = torch.sparse.FloatTensor(sparse_adj, torch.ones(5), torch.Size([3,5])).to_dense()
ind = dense_adj.nonzero(as_tuple=False).t().contiguous()
sparse_adj = torch.stack((ind[1], ind[0]), dim=0)

Or is there any alternative way that is better?

Output:
You can acheive this by first constructing a sparse matrix with torch.sparse then converting it to a dense matrix. For this you will need to provide torch.sparse.FloatTensor a 2D tensor of indices, a tensor of values as well as a output size:
sparse_adj = torch.tensor([[0, 1, 2, 1, 0], [0, 1, 2, 3, 4]])
torch.sparse.FloatTensor(sparse_adj, torch.ones(5), torch.Size([3,5])).to_dense()

You can get the size of the output matrix dynamically with
sparse_adj.max(axis=1).values + 1

So it becomes:
torch.sparse.FloatTensor(
    sparse_adj,
    torch.ones(sparse_adj.shape[1]),
    (sparse_adj.max(axis=1).values + 1).tolist())


Instruction: Pytorch UnpicklingError: A load persistent id instruction was encountered

Input:
My dataloader raises this error when loading its files:
UnpicklingError                           Traceback (most recent call last)
<ipython-input-14-cb081a68afbe> in <module>
----> 1 torch.load("/network/tmp1/ccai/data/labelbox_2020/imgs/AB_304.png")

~/.conda/envs/omnienv/lib/python3.8/site-packages/torch/serialization.py in load(f, map_location, pickle_module, **pickle_load_args)
    593                     return torch.jit.load(opened_file)
    594                 return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
--> 595         return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
    596
    597

~/.conda/envs/omnienv/lib/python3.8/site-packages/torch/serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

What's bugging me is that I'm not doing multiple loads from the same process as pointed out here.
Could it be that this error is triggered by multiple python processes reading the same file?
pytorch 1.7 on ubuntu 18 with python 3.8

Output: I found it: this error can be triggered when using torch.load on the wrong type of data, a .png image in this case.


Instruction: How to understand "torch.randn()" size* parameter arguments?

Input:
From what I understand, torch.randn(layers/depth, rows, columns), which can be seen when executing: torch.randn(2, 3, 3) ==> 2 layers (3x3) matrix:
tensor([[[ 1.4838,  1.2926,  1.6147],
     [ 0.7923,  0.6414, -0.2676],
     [-0.1949,  0.3859, -0.6940]],

    [[ 0.2454, -1.9215, -0.3078],
     [ 0.8544,  0.9726,  0.0330],
     [ 0.3579,  0.8247,  2.1288]]])

But what does adding an extra term in the size* parameter imply? As in: torch.randn(2, 1, 3, 3)
tensor([[[[ 0.6206, -1.3697, -0.2267],
      [ 1.0511,  2.3375, -0.9598],
      [-0.8148, -0.0911, -2.1211]]],

    [[[ 0.0659,  1.0764,  0.6150],
      [-1.7226,  0.5038, -0.9544],
      [-0.6447, -0.3325,  0.2048]]]])

What did the "1" add into the Tensor created?

Output:
Each number you are introducing refers to a dimension of the matrix. It is hard for humans to visualize more than 3 dimensions, but computers are fine with it.
In this particular case, you can think about the extra dimension as something like a batch size.


Instruction: Is the differentiation correct in pytorch tutorial?

Input:
I'm going through https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html#sphx-glr-beginner-blitz-autograd-tutorial-py and noticed an equation I couldn't understand. For the below equation for the tutorial,

Isn't do/dx = 6 (xi+2). How is it 3/2(xi+2)?

Output: There is a 1/4 term in front of the summation. So you get 6/4(xi + 2) = 3/2(xi + 2).


Instruction: How to get index.cuda?

Input:
I am reading a code about CMC downloaded from github, and it can't work on vscode.
Code:
if torch.cuda.is_available():
        index = index.cuda(async=True)
        inputs = inputs.cuda()

Error message is as follows:
  File "e:\CMC-master\train_CMC.py", line 219
    index = index.cuda(async=True)
                       ^
SyntaxError: invalid syntax

How to fix it?

Output:
Try using non_blocking=True instead:
index = index.cuda(non_blocking=True)

See Tensor.cuda for more information, and this answer.


Instruction: How to interpret Tensor sizes?

Input:
I'm having difficulty understanding the difference between the following:
x1 = torch.tensor([1, 2, 3])  # single brackets
x2 = torch.tensor([[1, 2, 3]])  # double brackets
When checking their sizes:
x1.size() and x2.size() we get the following:
torch.Size([3])
torch.Size([1, 3])

Which I interpret as x1 being a (3x1) column vector, while x2 is a (1x3) row vector.
However, when attempting to transpose both vectors:
print(x1.T)
print(x2.T), we get:
tensor([1, 2, 3])
tensor([[1],
        [2],
        [3]])

x1 seems to be unaffected by transposition?
Further when attempting to force x1 to be a (1x3) row vector using ".view()":
print(x1.view(1, -1)) we get:
tensor([[1, 2, 3]])  # double brackets

So how come ".T" didn't do the trick, but ".view(1, -1)" was able to transform x1 into a (1x3) row vector?
What really is x1 when we first assigned it?

Output:
As per official documentation -

Expects input to be <= 2-D tensor and transposes dimensions 0 and 1.
0-D and 1-D tensors are returned as is. When input is a 2-D tensor this is equivalent to transpose(input, 0, 1).

x = torch.randn(())
torch.t(x)
#tensor(0.1995)

x = torch.randn(3)
x
#tensor([ 2.4320, -0.4608,  0.7702])

torch.t(x)
#tensor([ 2.4320, -0.4608,  0.7702])

x = torch.randn(2, 3)
x
#tensor([[ 0.4875,  0.9158, -0.5872],
#        [ 0.3938, -0.6929,  0.6932]])

torch.t(x)
#tensor([[ 0.4875,  0.3938],
#        [ 0.9158, -0.6929],
#        [-0.5872,  0.6932]])

This is the reason why x1 has no effect. It's currently a 1D tensor and NOT a 2D tensor. There is a difference between the shape of (3,) and (3,1). The first only has a single axis while the other has 2 axis (similar to the double brackets you added)
This statement, Which I interpret as x1 being a (3x1) column vector, while x2 is a (1x3) row vector. is incorrect to some extent.
x1 #(3,) 1D tensor
x1.reshape((3,1) #(3,1) #2D tensor

x1.T #(1,3) 2D tensor with successful transpose


Instruction: Trying to accumulate gradients in Pytorch, but getting RuntimeError when calling loss.backward

Input:
I'm trying to train a model in Pytorch, and I'd like to have a batch size of 8, but due to memory limitations, I can only have a batch size of at most 4. I've looked all around and read a lot about accumulating gradients, and it seems like the solution to my problem.
However, I seem to have trouble implementing it. Every time I run the code I get RuntimeError: Trying to backward through the graph a second time. I don't understand why since my code looks like all these other examples I've seen (unless I'm just missing something major):

https://stackoverflow.com/a/62076913/1227353
https://medium.com/huggingface/training-larger-batches-practical-tips-on-1-gpu-multi-gpu-distributed-setups-ec88c3e51255
https://discuss.pytorch.org/t/why-do-we-need-to-set-the-gradients-manually-to-zero-in-pytorch/4903/20

One caveat is that the labels for my images are all different size, so I can't send the output batch and the label batch into the loss function; I have to iterate over them together. This is what an epoch looks like (it's been pared down for the sake of brevity):
  # labels_batch contains labels of different sizes
  for batch_idx, (inputs_batch, labels_batch) in enumerate(dataloader):
    outputs_batch = model(inputs_batch)

    # have to do this because labels can't be stacked into a tensor
    for output, label in zip(outputs_batch, labels_batch):
      output_scaled = interpolate(...)  # make output match label size
      loss = train_criterion(output_scaled, label) / (BATCH_SIZE * 2)
      loss.backward()

    if batch_idx % 2 == 1:
      optimizer.step()
      optimizer.zero_grad()

Is there something I'm missing? If I do the following I also get an error:
  # labels_batch contains labels of different sizes
  for batch_idx, (inputs_batch, labels_batch) in enumerate(dataloader):
    outputs_batch = model(inputs_batch)

    # CHANGE: we're gonna accumulate losses manually
    batch_loss = 0

    # have to do this because labels can't be stacked into a tensor
    for output, label in zip(outputs_batch, labels_batch):
      output_scaled = interpolate(...)  # make output match label size
      loss = train_criterion(output_scaled, label) / (BATCH_SIZE * 2)
      batch_loss += loss # CHANGE: accumulate!

    # CHANGE: do backprop outside for loop
    batch_loss.backward()

    if batch_idx % 2 == 1:
      optimizer.step()
      optimizer.zero_grad()

The error I get in this case is RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn. This happens when the next epoch starts though...   (INCORRECT, SEE EDIT BELOW)
How can I train my model with gradient accumulation? Or am I doomed to train with a batch size of 4 or less?
Oh and as a side question, does the location of where I put loss.backward() affect what I need to normalize the loss by? Or is it always normalized by BATCH_SIZE * 2?
EDIT:
The second code segment was getting an error due to the fact that I was doing torch.set_grad_enabled(phase == 'train') but I had forgotten to wrap the call to batch_loss.backward() with an if phase == 'train'... my bad
So now the second segment of code seems to work and do gradient accumulation, but why doesn't the first bit of code work? It feel equivalent to setting BATCH_SIZE as 1. Furthermore, I'm creating a new loss object each time, so shouldn't the calls to backward() operate on different graphs entirely?

Output:
It seems you have two issues here, you said you couldn't have batch_size=8 because of memory limitations but later state that your labels are not of  the same size. The latter seems much more important than the former. Anyway, I will try to answer your questions best I can.

How can I train my model with gradient accumulation? Or am I doomed to train with a batch size of 4 or less?

You want to call .backward() on every loop cycle otherwise the batch will have no effect on the training. You can then call step() and zero_grad() only when batch_idx % 2 is True (i.e. for every other batch).
Here's an example which accumulates the gradient, not the loss:
model = nn.Linear(10, 3)
optim = torch.optim.SGD(model.parameters(), lr=0.1)

ds = TensorDataset(torch.rand(100, 10), torch.rand(100, 3))
dl = DataLoader(ds, batch_size=4)

for i, (x, y) in enumerate(dl):
    y_hat = model(x)
    loss = F.l1_loss(y_hat, y) / 2
    loss.backward()

    if i % 2:
        optim.step()
        optim.zero_grad()

Note this approach is different to accumulating the loss, and back-propagating only all batches (or part of the batches) have gone through the network. In the example above we backpropagate every 4 datapoints and updating the model every 8 datapoints.

Oh and as a side question, does the location of where I put loss.backward() affect what I need to normalize the loss by? Or is it always normalized by BATCH_SIZE * 2?

Usually torch's built-in losses have reduction='mean' set as default. This means the loss gets averaged over all batch elements that contributed to calculating the loss. So this will depend on your loss implementation.
However if you are using gradient accumalation, then yes you will need to average your loss by the number of accumulation steps (here loss = F.l1_loss(y_hat, y) / 2). Since your gradients will be accumulated twice.
To read more about this, I recommend taking a look at this other SO post.


Instruction: torch.unique does not work for float tensors

Input:
I am trying to extract the unique elements from a float tensor.
I have tried :
out = torch.unique(my_tensor)

However this method only works for int/long tensor. My tensor is quantizied tensor in a non-uniform way, thus its guaranteed to have a small set of float values.

Output:
You could using numpy.unique instead
import torch
import numpy as np

t = torch.tensor([1.05, 1.05, 2.01, 2.01, 3.9, 3.9001])
print(np.unique(t.numpy()))

Outputs:
[1.05   2.01   3.9    3.9001]


Instruction: Efficient pytorch broadcasting not found

Input:
I have the following code snippet in my implemenatation. There is a nested for loop with 3 loops. In the main code the 3D coordinates of the original system is stacked as a 1D vector of constinous stacking of points as for a point with coordinate (x,y,z) a sample cells will look like
Predictions =[...x,y,z,...]

whereas for my calulation I need reshaped_prediction vector as a 2D matrix with prediction_reshaped[i][0]=x, prediction_reshaped[i][1]=y prediction_reshaped[i][2]=z
where i is any sample row in the matrix prediction_reshaped.
The following code shows the logic
prediction_reshaped=torch.zeros([batch,num_node,dimesion])
    for i in range(batch):
        for j in range(num_node):
            for k in range(dimesion):
                prediction_reshaped[i][j][k]=prediction[i][3*j+k]

is their any efficient broadcasting to avoid these three nested loop? it is slowing down my code. torch.reshape does not suit my purpose.
The code is implemented using pytorch with all matrices as pytorch tensor but any numpy solution will also help.

Output:
This should do the job.
import torch
batch = 2
num_nodes = 4
x = torch.rand(batch, num_nodes * 3)
# tensor([[0.8076, 0.2572, 0.7100, 0.4180, 0.6420, 0.4668, 0.8915, 0.0366,  0.5704,
#          0.0834, 0.3313, 0.9080],
#         [0.2925, 0.7367, 0.8013, 0.4516, 0.5470, 0.5123, 0.1929, 0.4191,  0.1174,
#          0.0076, 0.2864, 0.9151]])
x = x.reshape(batch, num_nodes, 3)
# tensor([[[0.8076, 0.2572, 0.7100],
#         [0.4180, 0.6420, 0.4668],
#         [0.8915, 0.0366, 0.5704],
#         [0.0834, 0.3313, 0.9080]],
#
#        [[0.2925, 0.7367, 0.8013],
#         [0.4516, 0.5470, 0.5123],
#         [0.1929, 0.4191, 0.1174],
#         [0.0076, 0.2864, 0.9151]]])


Instruction: How to use 'collate_fn' with dataloaders?

Input:
I am trying to train a pretrained roberta model using 3 inputs, 3 input_masks and a label as tensors of my training dataset.
I do this using the following code:
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
batch_size = 32
# Create the DataLoader for our training set.
train_data = TensorDataset(train_AT, train_BT, train_CT, train_maskAT, train_maskBT, train_maskCT, labels_trainT)
train_dataloader = DataLoader(train_data, batch_size=batch_size)

# Create the Dataloader for our validation set.
validation_data = TensorDataset(val_AT, val_BT, val_CT, val_maskAT, val_maskBT, val_maskCT, labels_valT)
val_dataloader = DataLoader(validation_data, batch_size=batch_size)

# Pytorch Training
training_args = TrainingArguments(
    output_dir='C:/Users/samvd/Documents/Master/AppliedMachineLearning/FinalProject/results',          # output directory
    num_train_epochs=1,              # total # of training epochs
    per_device_train_batch_size=32,  # batch size per device during training
    per_device_eval_batch_size=32,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_dir='C:/Users/samvd/Documents/Master/AppliedMachineLearning/FinalProject/logs',            # directory for storing logs
)

trainer = Trainer(
    model=model,                          # the instantiated  Transformers model to be trained
    args=training_args,                   # training arguments, defined above
    train_dataset = train_data,           # training dataset
    eval_dataset = validation_data,       # evaluation dataset
)

trainer.train()

However this gives me the following error:

TypeError: vars() argument must have dict attribute

Now I have found out that it is probably because I don't use collate_fn when using DataLoader, but I can't really find a source that helps me define this correctly so the trainer understands the different tensors I put in.
Can anyone point me in the right direction?

Output:
Basically, the collate_fn receives a list of tuples if your __getitem__ function from a Dataset subclass returns a tuple, or just a normal list if your Dataset subclass returns only one element. Its main objective is to create your batch without spending much time implementing it manually. Try to see it as a glue that you specify the way examples stick together in a batch. If you don’t use it, PyTorch only put batch_size examples together as you would using torch.stack (not exactly it, but it is simple like that).
Suppose for example, you want to create batches of a list of varying dimension tensors. The below code pads sequences with 0 until the maximum sequence size of the batch, that is why we need the collate_fn, because a standard batching algorithm (simply using torch.stack) won’t work in this case, and we need to manually pad different sequences with variable length to the same size before creating the batch.
def collate_fn(data):
    """
       data: is a list of tuples with (example, label, length)
             where 'example' is a tensor of arbitrary shape
             and label/length are scalars
    """
    _, labels, lengths = zip(*data)
    max_len = max(lengths)
    n_ftrs = data[0][0].size(1)
    features = torch.zeros((len(data), max_len, n_ftrs))
    labels = torch.tensor(labels)
    lengths = torch.tensor(lengths)

    for i in range(len(data)):
        j, k = data[i][0].size(0), data[i][0].size(1)
        features[i] = torch.cat([data[i][0], torch.zeros((max_len - j, k))])

    return features.float(), labels.long(), lengths.long()

The function above is fed to the collate_fn param in the DataLoader, as this example:
DataLoader(toy_dataset, collate_fn=collate_fn, batch_size=5)

With this collate_fn function, you always gonna have a tensor where all your examples have the same size. So, when you feed your forward() function with this data, you need to use the length to get the original data back, to not use those meaningless zeros in your computation.
Source: Pytorch Forum


Instruction: How to add a multiclass multilabel layer on top of pretrained BERT model?

Input:
I am trying to do a multitask multiclass sentence classification task using the pretrained BERT model from the huggingface transformers library . I have tried to use the BERTForSequenceClassification model from there but the issue I am having is that I am not able to extend it for multiple tasks . I will try to make it more informative through this example.
Suppose we have four different tasks and for each sentence and for each task we have labels like this as follows in the examples:

A :[ 'a' , 'b' , 'c' , 'd' ]
B :[ 'e' , 'f' , 'g' , 'h' ]
C :[ 'i' , 'j' , 'k' , 'l' ]
D :[ 'm' , 'n' , 'o' , 'p' ]

Now , if I have a sentence for this model , I want the output to give me output for all the four different tasks (A,B,C,D).
This is what I was doing earlier
   model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased", # Use the 12-layer BERT model, with an uncased vocab.
    num_labels = 4, # The number of output labels--2 for binary classification.
                    # You can increase this for multi-class tasks.
    output_attentions = False, # Whether the model returns attentions weights.
    output_hidden_states = False, # Whether the model returns all hidden-states.
)

Then I tried to implement a CustomBERT model like this :
class CustomBERTModel(nn.Module):
    def __init__(self):
          super(CustomBERTModel, self).__init__()
          self.bert = BertModelForSequenceClassification.from_pretrained("bert-base-uncased")
          ### New layers:
          self.linear1 = nn.Linear(768, 256)
          self.linear2 = nn.Linear(256, num_classes) ## num_classes is the number of classes in this example

    def forward(self, ids, mask):
          sequence_output, pooled_output = self.bert(
               ids,
               attention_mask=mask)

          # sequence_output has the following shape: (batch_size, sequence_length, 768)
          linear1_output = self.linear1(sequence_output[:,0,:].view(-1,768))
          linear2_output = self.linear2(linear2_output)

          return linear2_output

I have went through the answers to questions similar to it available earlier but none of them appeared to
answer my question . I have tried to get through all the points which I think can be helpful for the understanding of my problem and would try to clear further more in case of any descrepancies made by me in the explaination of the question . Any answers related to this will be very much helpful .

Output:
You should use BertModel and not BertModelForSequenceClassification, as BertModelForSequenceClassification adds a linear layer for classification on top of BERT model and uses CrossEntropyLoss, which is meant for multiclass classification.
Hence, first use BertModel instead of BertModelForSequenceClassification:
class CustomBERTModel(nn.Module):
    def __init__(self):
          super(CustomBERTModel, self).__init__()
          self.bert = BertModel.from_pretrained("bert-base-uncased")
          ### New layers:
          self.linear1 = nn.Linear(768, 256)
          self.linear2 = nn.Linear(256, 4) ## as you have 4 classes in the output
          self.sig = nn.functional.sigmoid()

    def forward(self, ids, mask):
          sequence_output, pooled_output = self.bert(
               ids,
               attention_mask=mask)

          # sequence_output has the following shape: (batch_size, sequence_length, 768)
          linear1_output = self.linear1(sequence_output[:,0,:].view(-1,768))
          linear2_output = self.linear2(linear2_output)
          linear2_output = self.sig(linear2_output)

          return linear2_output

Next, multilabel classification uses 'Sigmoid' activation instead of 'Softmax' (Here, the sigmoid layer is added in the above code)
Further, for multilabel classification, you need to use BCELoss instead of CrossEntropyLoss.


Instruction: PyTorch - ModuleNotFoundError

Input:
I've trained Retinanet for object detection in google colab and now I want to load its .pt file in another python project but I keep getting this error. Any thoughts?
Traceback (most recent call last):
  File "C:\Users\stefan_cepa995\Desktop\breast-mammography-app\app.py", line 522, in <module>
    model = torch.load(os.path.join(".", "models", "retinanet", "retinanet_gwd.pt"))
  File "C:\Users\stefan_cepa995\anaconda3\envs\tensorflow\lib\site-packages\torch\serialization.py", line 594, in load
    return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
  File "C:\Users\stefan_cepa995\anaconda3\envs\tensorflow\lib\site-packages\torch\serialization.py", line 853, in _load
    result = unpickler.load()
ModuleNotFoundError: No module named 'retinanet'

Output: As @Rika mentioned in comments, solution is to save state_dict and then load it with load_state_dict() function.