Instruction: EasyOCR used under Python / Torch Multiprocessing is defaulting to CPU

Input:
I am using EasyOCR for text extraction from images. It uses PyTorch. There are multiple images in different folders and the sequence in which these folders are read isn't consequential.
When run in sequence, EasyOCR is by default using GPU and is faster compared to when run on CPU. But when Python / Torch Multiprocessing is invoked, so that the multiple folders are read in parallel, EasyOCR is defaulting to CPU.
torch.cuda.is_available returns False.
How can I solve this?

Output:
If torch.cuda.is_available returns False,

Verify your device has a GPU.
Verify that the installed version of CUDA is supported on your GPU.
Verify that you have installed torch with CUDA support.

Check this question for additional details:
Why `torch.cuda.is_available()` returns False even after installing pytorch with cuda?


Instruction: "Didn't find engine for operation quantized" error while using dynamic quantization with Huggingface transformer

Input:
I am trying to do dynamic quantization(quantizes the weights and the activations) on a pytorch pre-trained model from huggingface library. I have referred this link and found dynamic quantization the most suitable. I will be using the quantized model on a CPU.
Link to hugginface model here.
torch version: 1.6.0 (installed via pip)
Pre-trained models
tokenizer = AutoTokenizer.from_pretrained("microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext")
model = AutoModel.from_pretrained("microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext")

Dynamic quantization
quantized_model = torch.quantization.quantize_dynamic(
    model, qconfig_spec={torch.nn.Linear}, dtype=torch.qint8
)

print(quantized_model)

Error
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-7-df2355c17e0b> in <module>
      1 quantized_model = torch.quantization.quantize_dynamic(
----> 2     model, qconfig_spec={torch.nn.Linear}, dtype=torch.qint8
      3 )
      4
      5 print(quantized_model)

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in quantize_dynamic(model, qconfig_spec, dtype, mapping, inplace)
    283     model.eval()
    284     propagate_qconfig_(model, qconfig_spec)
--> 285     convert(model, mapping, inplace=True)
    286     _remove_qconfig(model)
    287     return model

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    363     for name, mod in module.named_children():
    364         if type(mod) not in SWAPPABLE_MODULES:
--> 365             convert(mod, mapping, inplace=True)
    366         reassign[name] = swap_module(mod, mapping)
    367

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    363     for name, mod in module.named_children():
    364         if type(mod) not in SWAPPABLE_MODULES:
--> 365             convert(mod, mapping, inplace=True)
    366         reassign[name] = swap_module(mod, mapping)
    367

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    363     for name, mod in module.named_children():
    364         if type(mod) not in SWAPPABLE_MODULES:
--> 365             convert(mod, mapping, inplace=True)
    366         reassign[name] = swap_module(mod, mapping)
    367

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    363     for name, mod in module.named_children():
    364         if type(mod) not in SWAPPABLE_MODULES:
--> 365             convert(mod, mapping, inplace=True)
    366         reassign[name] = swap_module(mod, mapping)
    367

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    363     for name, mod in module.named_children():
    364         if type(mod) not in SWAPPABLE_MODULES:
--> 365             convert(mod, mapping, inplace=True)
    366         reassign[name] = swap_module(mod, mapping)
    367

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in convert(module, mapping, inplace)
    364         if type(mod) not in SWAPPABLE_MODULES:
    365             convert(mod, mapping, inplace=True)
--> 366         reassign[name] = swap_module(mod, mapping)
    367
    368     for key, value in reassign.items():

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/quantization/quantize.py in swap_module(mod, mapping)
    393             )
    394             device = next(iter(devices)) if len(devices) > 0 else None
--> 395             new_mod = mapping[type(mod)].from_float(mod)
    396             if device:
    397                 new_mod.to(device)

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/nn/quantized/dynamic/modules/linear.py in from_float(cls, mod)
    101         else:
    102             raise RuntimeError('Unsupported dtype specified for dynamic quantized Linear!')
--> 103         qlinear = Linear(mod.in_features, mod.out_features, dtype=dtype)
    104         qlinear.set_weight_bias(qweight, mod.bias)
    105         return qlinear

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/nn/quantized/dynamic/modules/linear.py in __init__(self, in_features, out_features, bias_, dtype)
     33
     34     def __init__(self, in_features, out_features, bias_=True, dtype=torch.qint8):
---> 35         super(Linear, self).__init__(in_features, out_features, bias_, dtype=dtype)
     36         # We don't muck around with buffers or attributes or anything here
     37         # to keep the module simple. *everything* is simply a Python attribute.

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/nn/quantized/modules/linear.py in __init__(self, in_features, out_features, bias_, dtype)
    150             raise RuntimeError('Unsupported dtype specified for quantized Linear!')
    151
--> 152         self._packed_params = LinearPackedParams(dtype)
    153         self._packed_params.set_weight_bias(qweight, bias)
    154         self.scale = 1.0

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/nn/quantized/modules/linear.py in __init__(self, dtype)
     18         elif self.dtype == torch.float16:
     19             wq = torch.zeros([1, 1], dtype=torch.float)
---> 20         self.set_weight_bias(wq, None)
     21
     22     @torch.jit.export

~/.virtualenvs/python3/lib64/python3.6/site-packages/torch/nn/quantized/modules/linear.py in set_weight_bias(self, weight, bias)
     24         # type: (torch.Tensor, Optional[torch.Tensor]) -> None
     25         if self.dtype == torch.qint8:
---> 26             self._packed_params = torch.ops.quantized.linear_prepack(weight, bias)
     27         elif self.dtype == torch.float16:
     28             self._packed_params = torch.ops.quantized.linear_prepack_fp16(weight, bias)

RuntimeError: Didn't find engine for operation quantized::linear_prepack NoQEngine

Output:
Is qnnpack in the list when you run print(torch.backends.quantized.supported_engines)?
Does torch.backends.quantized.engine = 'qnnpack' work for you?


Instruction: pipenv install pytorch cpu + specific version

Input:
I ned to install a specific version of pytorch cpu mode.
With pip I would do it like this:
pip install torch==1.2.0+cpu torchvision==0.4.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --trusted-host download.pytorch.org
How can I achieve the same using Pipenv?
I tried having the following Pipfile:
[[source]]
name = "pytorch"
url = "https://download.pytorch.org/whl/cpu"
verify_ssl = false

[packages]
torchvision = {index = "pytorch", version = "==0.4.0"}
torch = {index = "pytorch", version = "==1.2.0"}

but didn't work

Output:
You can do:
$ PIP_FIND_LINKS="https://download.pytorch.org/whl/torch_stable.html" pipenv install torch==1.2.0+cpu torchvision==0.4.0+cpu

But, you'll have to ensure that you add PIP_FIND_LINKS for any consecutive pipenv sync, pipenv lock, etc.
UPD:
You may also add PIP_FIND_LINKS="https://download.pytorch.org/whl/torch_stable.html" to the .env file, but it's only being loaded on pipenv run and pipenv shell.


Instruction: PyTorch installation fails Could not find a version that satisfies the requirement

Input:
I'm trying to install PyTorch with PyCharm Community Edition 2020.2.3 x64 and Python 3.9.0 on Windows 10 pro 64-bit OS PC machine
I've tried:
pip install torch==1.7.0+cpu torchvision==0.8.1+cpu torchaudio===0.7.0 -f https://download.pytorch.org/whl/torch_stable.html

and:
python -m pip install torch==1.7.0 -f https://download.pytorch.org/whl/torch_stable.html

Should I downgrade Python version, let's say to Python 3.8.6 or PyTorch version, to make it work, or am I doing something else incorrectly  besides this, maybe missed something to install, for example I did not selected CUDA, but seems like it is different reason:

ERROR: Could not find a version that satisfies the requirement
torch==1.7.0+cpu (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2)
ERROR: No matching distribution found for torch==1.7.0+cpu

with pip3 install https://download.pytorch.org/whl/cpu//torch-1.7.0%2Bcpu-cp38-cp38-win_amd64.whl:

ERROR: torch-1.7.0+cpu-cp38-cp38-win_amd64.whl is not a supported
wheel on this platform.

and
pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html -vvv

ERROR: Could not find a version that satisfies the requirement
torch==1.4.0+cpu (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2)
ERROR: No matching distribution found for torch==1.4.0+cpu

Any advice, guide or example would be helpful
Solution:
Installed successfully with Python 3.8.6

Output: I tried with Python 3.8.0, Python 3.8.5, Python 3.8.6 and Python 3.9.0. It seems to work only with 3.8.6 version.


Instruction: PyTorch how to compute second order Jacobian?

Input:
I have a neural network that's computing a vector quantity u. I'd like to compute first and second-order jacobians with respect to the input x, a single element.
Would anybody know how to do that in PyTorch? Below, the code snippet from my project:
import torch
import torch.nn as nn

class PINN(torch.nn.Module):

    def __init__(self, layers:list):
        super(PINN, self).__init__()
        self.linears = nn.ModuleList([])
        for i, dim in enumerate(layers[:-2]):
            self.linears.append(nn.Linear(dim, layers[i+1]))
            self.linears.append(nn.ReLU())
        self.linears.append(nn.Linear(layers[-2], layers[-1]))

    def forward(self, x):
        for layer in self.linears:
            x = layer(x)
        return x

I then instantiate my network:
n_in = 1
units = 50
q = 500

pinn = PINN([n_in, units, units, units, q+1])
pinn

Which returns
PINN(
  (linears): ModuleList(
    (0): Linear(in_features=1, out_features=50, bias=True)
    (1): ReLU()
    (2): Linear(in_features=50, out_features=50, bias=True)
    (3): ReLU()
    (4): Linear(in_features=50, out_features=50, bias=True)
    (5): ReLU()
    (6): Linear(in_features=50, out_features=501, bias=True)
  )
)

Then I compute both FO and SO jacobians
x = torch.randn(1, requires_grad=False)

u_x = torch.autograd.functional.jacobian(pinn, x, create_graph=True)
print("First Order Jacobian du/dx of shape {}, and features\n{}".format(u_x.shape, u_x)

u_xx = torch.autograd.functional.jacobian(lambda _: u_x, x)
print("Second Order Jacobian du_x/dx of shape {}, and features\n{}".format(u_xx.shape, u_xx)

Returns
First Order Jacobian du/dx of shape torch.Size([501, 1]), and features
tensor([[-0.0310],
        [ 0.0139],
        [-0.0081],
        [-0.0248],
        [-0.0033],
        [ 0.0013],
        [ 0.0040],
        [ 0.0273],
        ...
        [-0.0197]], grad_fn=<ViewBackward>)

Second Order Jacobian du/dx of shape torch.Size([501, 1, 1]), and features
tensor([[[0.]],

        [[0.]],

        [[0.]],

        [[0.]],

        ...

        [[0.]]])

Should not u_xx be a None vector if it didn't depend on x?
Thanks in advance

Output:
So as @jodag mentioned in his comment, ReLU being null or linear, its gradient is constant (except on 0, which is a rare event), so its second-order derivative is zero. I changed the activation function to Tanh, which finally allows me to compute the jacobian twice.
Final code is
import torch
import torch.nn as nn

class PINN(torch.nn.Module):

    def __init__(self, layers:list):
        super(PINN, self).__init__()
        self.linears = nn.ModuleList([])
        for i, dim in enumerate(layers[:-2]):
            self.linears.append(nn.Linear(dim, layers[i+1]))
            self.linears.append(nn.Tanh())
        self.linears.append(nn.Linear(layers[-2], layers[-1]))

    def forward(self, x):
        for layer in self.linears:
            x = layer(x)
        return x

    def compute_u_x(self, x):
        self.u_x = torch.autograd.functional.jacobian(self, x, create_graph=True)
        self.u_x = torch.squeeze(self.u_x)
        return self.u_x

    def compute_u_xx(self, x):
        self.u_xx = torch.autograd.functional.jacobian(self.compute_u_x, x)
        self.u_xx = torch.squeeze(self.u_xx)
        return self.u_xx

Then calling compute_u_xx(x) on an instance of PINN with x.require_grad set to True gets me there. How to get rid of useless dimensions introduced by torch.autograd.functional.jacobian remains to be understood though...


Instruction: DenseNet, Sizes of tensors must match

Input:
would you know how I can adapt this code so that sizes of tensors must match because I have this error: x = torch.cat([x1,x2],1) RuntimeError: Sizes of tensors must match except in dimension 0. Got 32 and 1 (The offending index is 0).
My images are size 416x416.
Thank you in advance for your help,
num_classes = 20
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.inc = models.inception_v3(pretrained=True)
        self.inc.aux_logits = False

        for child in list(self.inc.children())[:-5]:
            for param in child.parameters():
                param.requires_grad = False

        self.inc.fc = nn.Sequential()

        self.dens121 = models.densenet121(pretrained=True)

        for child in list(self.dens121.children())[:-6]:
            for param in child.parameters():
                param.requires_grad = False

        self.dens121 = nn.Sequential(*list(self.dens121.children())[:-1])

        self.SiLU = nn.SiLU()
        self.linear = nn.Linear(4096, num_classes)
        self.dropout = nn.Dropout(0.2)

    def forward(self, x):
        x1 = self.SiLU(self.dens121(x))
        x1 = x1.view(-1, 2048)

        x2 = self.inc(x).view(-1, 2048)
        x = torch.cat([x1,x2],1)

        return self.linear(self.dropout(x))

Output:
The shapes of the two tensors are very different and that's why the torch.cat() fails. I tried to run your code with the following example:
def forward(self, x):
    x1 = self.SiLU(self.dens121(x))
    x1 = x1.view(-1, 2048)

    x2 = self.inc(x).view(-1, 2048)
    print(x1.shape, x2.shape)
    x = torch.cat([x1,x2], dim=1)

    return self.linear(self.dropout(x))

Here's the driver code
inputs = torch.randn(2, 3, 416, 416)
model = Net()
outputs = model(inputs)

The shapes of x1 of x2 are as follows:
torch.Size([169, 2048]) torch.Size([2, 2048])

Either your DenseNet should output the same shape as the output of Inceptionv3 or vice-versa. The output from DenseNet is of shape torch.Size([2, 1024, 13, 13]) and the output from Inceptionv3 is of shape torch.Size([2, 2048]).
EDIT
Add this line to the init method:
self.conv_reshape= nn.Conv2d(1024, 2048, kernel_size=13, stride=1)

Add these lines to your forward():
x1 = self.SiLU(self.dens121(x))

out = self.conv_reshape(x1)
x1 = out.view(-1, out.size(1))

x2 = self.inc(x).view(-1, 2048)


Instruction: The definition of "heads" in MultiheadAttention in Pytorch Transformer module

Input:
I am a bit confused about the definition of Multihead.
Are [1] and [2] below the same?
[1]
My understanding about multiplhead is the multiple attention patterns as below.
"multiple sets of Query/Key/Value weight matrices (the Transformer uses eight attention heads, so we end up with eight sets for each encoder/decoder)."
http://jalammar.github.io/illustrated-transformer/
But
[2] in class MultiheadAttention(Module): in Pytorch Transformer module,
it seems like embed_dim is DIVIDED by the number of heads.. WHy?
Or... the embed_dim is meant to be the feature dimension times the number of heads in the first place?
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/activation.py

Output:
As per your understanding, multi-head attention is multiple times attention over some data.
But on contrast, it isn't implemented by multiplying the set of weights into number of required attention. Instead, you rearrange the weight matrices corresponding to the number of attentions, that is reshape to the weight-matrix. So, in essence, it still remains multiple time attention, but you are attending different parts of the weights.


Instruction: "RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn " error BertFoeSequenceClassification

Input:
I am trying to build Bert model for Arabic Text classification task using pretrained model from https://github.com/alisafaya/Arabic-BERT
i want to know the exact difference between the two statement:
model_name = 'kuisailab/albert-large-arabic'
model = AutoModel.from_pretrained(model_name)
model = BertForSequenceClassification .from_pretrained(model_name)

I fine-tuned the model by adding the following layers on top of the model:
for param in model.parameters():
    param.requires_grad = False

model.classifier = nn.Sequential(
nn.Dropout(0.5),
nn.ReLU(),
nn.Linear(768,512),
nn.Linear(512,2),
nn.LogSoftmax(dim=1),
nn.Softmax(dim=1)
)
model = model.to(device)

and used the optimizer:
optimizer = AdamW(model.parameters(),
                  lr = 2e-5)

finally this is my training loop:
model.train()

for idx, row in train_data.iterrows():
    text_parts = preprocess_text(str(row['sentence']))
    label = torch.tensor([row['label']]).long().to(device)

    optimizer.zero_grad()

    overall_output = torch.zeros((1, 2)).float().to(device)
    for part in text_parts:
        if len(part) > 0:
            try:
                input = part.reshape(-1)[:512].reshape(1, -1)
                # print(input.shape)
                overall_output += model(input, labels=label)[1].float().to(device)
            except Exception as e:
                print(str(e))

#     overall_output /= len(text_parts)
    overall_output = F.softmax(overall_output[0], dim=-1)

    if label == 0:
        label = torch.tensor([1.0, 0.0]).float().to(device)
    elif label == 1:
        label = torch.tensor([0.0, 1.0]).float().to(device)

    # print(overall_output, label)

    loss = criterion(overall_output, label)
    total_loss += loss.item()

    loss.backward()
    optimizer.step()

and i get the error:
mat1 dim 1 must match mat2 dim 0
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-33-5c2f0fea6c1f> in <module>()
     39     total_loss += loss.item()
     40
---> 41     loss.backward()
     42     optimizer.step()
     43

1 frames
/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
    130     Variable._execution_engine.run_backward(
    131         tensors, grad_tensors_, retain_graph, create_graph,
--> 132         allow_unreachable=True)  # allow_unreachable flag
    133
    134

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

any idea how to solve this error

Output:
BertForSequenceClassification is the class which extends the BertModel, i.e, BertForSequenceClassification defines a logistic regression layer, for the task of classificiation, with cross-entropy loss, to be jointly fine-tuned or trained on the existing Bert Model.
AutoModel, is a class provided in the library that allows to automatically identify the model class based on it's name or the model file contents.
Since you already know that you need a model for classification, you can directly use BertForSequenceClassification


Instruction: How to compute Hessian of the loss w.r.t. the parameters in PyTorch using autograd.grad

Input:
I know there is quite a bit of content out there about "computing the Hessian" in pytorch, but as far as I've seen I haven't found anything working for me. So to try to be most precise, the Hessian that I want is the Jacobian of the gradient of the loss with respect to the network parameters. Also called the matrix of second-order derivatives with respect to the parameters.
I found some code that works in an intuitive way, although shouldn't be fast. It clearly just computes the gradient of the gradient of the loss w.r.t. the params w.r.t the params, and it does it one element (of the gradient) at a time. I think the logic is definitely right but I am getting an error, having to do with requires_grad. I'm a pytorch beginner so maybe its a simple thing, but the error seems to be saying that it can't take the gradient of the env_grads variable, which is the output from the previous grad function call.
Any help with this would be greatly appreciated. Here is the code followed by the error message. I also printed out the env_grads[0] variable so we can see that it is in fact a tensor, which is the correct output from the previous grad call.
env_loss = loss_fn(env_outputs, env_targets)
total_loss += env_loss
env_grads = torch.autograd.grad(env_loss, params,retain_graph=True)

print( env_grads[0] )
hess_params = torch.zeros_like(env_grads[0])
for i in range(env_grads[0].size(0)):
    for j in range(env_grads[0].size(1)):
        hess_params[i, j] = torch.autograd.grad(env_grads[0][i][j], params, retain_graph=True)[0][i, j] #  <--- error here
print( hess_params )
exit()

Output:
tensor([[-6.4064e-03, -3.1738e-03,  1.7128e-02,  8.0391e-03],
        [ 7.1698e-03, -2.4640e-03, -2.2769e-03, -1.0687e-03],
        [-3.0390e-04, -2.4273e-03, -4.0799e-02, -1.9149e-02],
        ...,
        [ 1.1258e-02, -2.5911e-05, -9.8133e-02, -4.6059e-02],
        [ 8.1502e-04, -2.5814e-03,  4.1772e-02,  1.9606e-02],
        [-1.0075e-02,  6.6072e-03,  8.3118e-04,  3.9011e-04]], device='cuda:0')

Error:
Traceback (most recent call last):
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 258, in <module>
    main(args)
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 245, in main
    deep_mask=args.deep_mask
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/run_synthetic.py", line 103, in train
    scale_grad_inverse_sparsity=scale_grad_inverse_sparsity
  File "/home/jefferythewind/Projects/Irina/learning-explanations-hard-to-vary/and_mask/and_mask_utils.py", line 154, in get_grads_deep
    hess_params[i, j] = torch.autograd.grad(env_grads[0][i][j], params, retain_graph=True)[0][i, j]
  File "/home/jefferythewind/anaconda3/envs/rapids3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 157, in grad
    inputs, allow_unused)
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

Output:
PyTorch recently-ish added a functional higher level API to torch.autograd which provides torch.autograd.functional.hessian(func, inputs,...) to directly evaluate the hessian of the scalar function func with respect to its arguments at a location specified by inputs, a tuple of tensors corresponding to the arguments of func. hessian itself does not support automatic differentiation, I believe.
Note, however, that as of March 2021 it is still in beta.

Full example using torch.autograd.functional.hessian to create a score-test for non-zero mean (As a (bad) alternative to the one sample t-test):
import numpy as np
import torch, torchvision
from torch.autograd import Variable, grad
import torch.distributions as td
import math
from torch.optim import Adam
import scipy.stats

x_data = torch.randn(100)+0.0 # observed data (here sampled under H0)

N = x_data.shape[0] # number of observations

mu_null = torch.zeros(1)
sigma_null_hat = Variable(torch.ones(1), requires_grad=True)

def log_lik(mu, sigma):
  return td.Normal(loc=mu, scale=sigma).log_prob(x_data).sum()

# Find theta_null_hat by some gradient descent algorithm (in this case an closed-form expression would be trivial to obtain (see below)):
opt = Adam([sigma_null_hat], lr=0.01)
for epoch in range(2000):
    opt.zero_grad() # reset gradient accumulator or optimizer
    loss = - log_lik(mu_null, sigma_null_hat) # compute log likelihood with current value of sigma_null_hat  (= Forward pass)
    loss.backward() # compute gradients (= Backward pass)
    opt.step()      # update sigma_null_hat

print(f'parameter fitted under null: sigma: {sigma_null_hat}, expected: {torch.sqrt((x_data**2).mean())}')
#> parameter fitted under null: sigma: tensor([0.9260], requires_grad=True), expected: 0.9259940385818481

theta_null_hat = (mu_null, sigma_null_hat)

U = torch.tensor(torch.autograd.functional.jacobian(log_lik, theta_null_hat)) # Jacobian (= vector of partial derivatives of log likelihood w.r.t. the parameters (of the full/alternative model)) = score
I = -torch.tensor(torch.autograd.functional.hessian(log_lik, theta_null_hat)) / N # estimate of the Fisher information matrix
S = torch.t(U) @ torch.inverse(I) @ U / N # test statistic, often named "LM" (as in Lagrange multiplier), would be zero at the maximum likelihood estimate

pval_score_test = 1 - scipy.stats.chi2(df = 1).cdf(S) # S asymptocially follows a chi^2 distribution with degrees of freedom equal to the number of parameters fixed under H0
print(f'p-value Chi^2-based score test: {pval_score_test}')
#> p-value Chi^2-based score test: 0.9203232752568568

# comparison with Student's t-test:
pval_t_test = scipy.stats.ttest_1samp(x_data, popmean = 0).pvalue
print(f'p-value Student\'s t-test: {pval_t_test}')
#> p-value Student's t-test: 0.9209265268946605


Instruction: Triton inference server serving TorchScript model

Input:
I am trying to serve a TorchScript model with the triton (tensorRT) inference server. But every time I start the server it throws the following error:
PytorchStreamReader failed reading zip archive: failed finding central directory

My folder structure is :
<model_repository>
  <model_name>
   config.pbtxt
   <1>
    <model.pt>

My config.pbtxt file is :
name: "model"
platform: "pytorch_libtorch"
max_batch_size: 1

input[
{
name: "INPUT__0"
data_type:  TYPE_FP32
dims: [-1,3,-1,-1]
}
]

output:[
{
name: "OUTPUT__0"
data_type:  TYPE_FP32
dims: [-1,1,-1,-1]
}
]

Output: I found the solution. It was a silly mistake on my part. The .pt torchscript file was not loaded properly.


Instruction: Run validation on 1 GPU while Train on multi-GPU Pytorch Lightning

Input:
Is there any way I can execute validation_step method on single GPU while training_step with multiple GPU using DDP.
The reason I want to do is because there are several metrics which I want to implement which requires complete access to the data, and running on single GPU will ensure that. I have tried validation_step_end method but somehow I am only getting part of the data. That post is here: Stack Overflow Post

Output: I am afraid that this is not possible. But there is the TorchMetrics package which has been developed with multi-GPU support in mind so when your custom metric is derived from TM you shall be able to get running even on your multi-GPU setting.


Instruction: How to use a learnable parameter in pytorch, constrained between 0 and 1?

Input:
I want to use a learnable parameter that only takes values between 0 and 1. How can I do this in pytorch?
Currently I am using:
self.beta = Parameter(torch.Tensor(1))
#initialize
zeros(self.beta)

But I am getting zeros and NaN for this parameter, as I train.

Output:
You can have a "raw" parameter taking any values, and then pass it through a sigmoid function to get a values in range (0, 1) to be used by your function.
For example:
class MyZeroOneLayer(nn.Module):
  def __init__(self):
    self.raw_beta = nn.Parameter(data=torch.Tensor(1), requires_grad=True)

  def forward(self):  # no inputs
    beta = torch.sigmoid(self.raw_beta)  # get (0,1) value
    return beta

Now you have a module with trainable parameter that is effectively in range (0,1)


Instruction: How to randomly mix two PyTorch tensors

Input: I have two same-shaped PyTorch tensors A and B, and I'd like to create a same-shape "randomly mixed" tensor C where C[i,...] = A[i,...] with probability alpha or B[i,...] with probability 1-alpha.  Is there some Pythonic way to do this compactly?

Output:
consider using torch.bernoulli to create a mask tensor:
import torch

prob = 0.8
x = torch.full((2, 6, 3), 10.2, dtype=torch.float)
y = torch.full((2, 6, 3), -1.6, dtype=torch.float)

mask = torch.bernoulli(torch.full(x.shape, prob)).int()
reverse_mask = torch.ones(x.shape).int() - mask

result = x * mask + y * reverse_mask

result is now:
[[[10.2000, 10.2000, 10.2000],
[10.2000, -1.6000, 10.2000],
[10.2000, 10.2000, -1.6000],
[-1.6000, 10.2000, -1.6000],
[10.2000, 10.2000, 10.2000],
[10.2000, 10.2000, 10.2000]],
[[10.2000, 10.2000, -1.6000],
[10.2000, 10.2000, 10.2000],
[10.2000, 10.2000, -1.6000],
[10.2000, -1.6000, 10.2000],
[-1.6000, 10.2000, 10.2000],
[10.2000, 10.2000, 10.2000]]]

Good Luck!


Instruction: RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.cuda.HalfTensor) should be the same

Input:
I just start to learn about the YOLO v5 PyTorch version and I was able to build a model, so then I tried to implement a flask application for real-time prediction using this trained model.
class for load model and predict
class Model(object):

    def __init__(self, model):

        self.device = torch_utils.select_device()
        print(self.device)
        model = torch.load(model, map_location=self.device)['model']

        self.half = False and self.device.type != 'cpu'
        print('half = ' + str(self.half))

        if self.half:
            model.half()

        # model  = model.to(self.device).eval()
        model.cuda()

        self.loaded_model = model

    def predict(self, img):
        global session
        # img1 = torch.from_numpy(img).to(self.device)
        # img = img1.reshape(1, 3, 640, 640)
        img = img.half() if self.half else img.float()  # uint8 to fp16/32
        img /= 255.0  # 0 - 255 to 0.0 - 1.0
        print(img.ndimension())
        if img.ndimension() == 3:
            img = img.unsqueeze(0)
        print(self.loaded_model)
        img = img.to(self.device)
        # img = img.half()
        self.preds = self.loaded_model(img, augment=False)[0]
        print(self.predict())
        return self.preds

Camera class for reading frames from camera or video
model = Model("weights/best.pt")
class Camera(object):
    def __init__(self):
        # self.video = cv2.VideoCapture('facial_exp.mkv')
        self.video = cv2.VideoCapture(0)

    def __del__(self):
        self.video.release()

    def get_frame(self):
        _, fr = self.video.read()
        loader = transforms.Compose([transforms.ToTensor()])

        image = cv2.resize(fr, (640, 640), interpolation=cv2.INTER_AREA)
        input_im = image.reshape(1, 640, 640, 3)

        pil_im = Image.fromarray(fr)
        image = loader(pil_im).float()
        # image = Variable(image, requires_grad=True)
        image = image.unsqueeze(0)

        pred = model.predict(input_im)
        pred = model.predict(image)
        print(pred)

        _, jpeg = cv2.imencode('.jpg', fr)
        return jpeg.tobytes()

Some of the commented lines are the ways which I tried but in all times bellow line
self.preds = self.loaded_model(img, augment=False)[0]  throws below error
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.cuda.HalfTensor) should be the same

any idea or guidance for solving this error thank you.

Output:
this error means: the input type is float32, the weight type(of your model) is float16.
for exsample, this code below runned:
model.half()  # so the weight type is float16
but this code below not runned:
img = img.half()  # so the input type is float32
please check your code.
for more information about 'half', you can refer to torch.Tensor.to() and torch.nn.Module.to()


Instruction: How to "cut" a tensor into half in Pytorch?

Input: I have a tensor with shape [1, 2, 96, 96] and would like two tensors with the shape [1, 1, 96, 96], is there a quick way of doing this? Thanks in advance

Output:
a, b = tensor.split(1, dim=1) should do the job. By specifying 1 you specify how many elements should be in each split e.g. [1,2,3,4,5,6].split(2) -> [1,2] [3,4] [5,6]. Then dim just specifies which dimension to split over which in your case would be one.
EDIT:
if you wanted to cut it in half more generally use tensor.split(n) where n is half the size of the tensor. So in your specific case if you had shape [1,10,96,96] you would use tensor.split(5,dim=1)


Instruction: What is the function of FrozenBatchNorm2d in “maskrcnn_benchmark”?

Input:
"maskrcnn_benchmark"s github
Here is the source code for "FrozenBatchNorm2d"
import torch
from torch import nn
class FrozenBatchNorm2d(nn.Module):
    def __init__(self, n):
        super(FrozenBatchNorm2d, self).__init__()
        self.register_buffer("weight", torch.ones(n))
        self.register_buffer("bias", torch.zeros(n))
        self.register_buffer("running_mean", torch.zeros(n))
        self.register_buffer("running_var", torch.ones(n))

    def forward(self, x):
        scale = self.weight * self.running_var.rsqrt()
        bias = self.bias - self.running_mean * scale
        scale = scale.reshape(1, -1, 1, 1)
        bias = bias.reshape(1, -1, 1, 1)
        return x * scale + bias

When I put this function in my script, I found that this function had almost no effect.
Here is my usage
import torch.nn as nn
import torch
class FrozenBatchNorm2d(nn.Module):
    """
    BatchNorm2d where the batch statistics and the affine parameters
    are fixed
    """

    def __init__(self, n):
        super(FrozenBatchNorm2d, self).__init__()
        self.register_buffer("weight", torch.ones(n))
        self.register_buffer("bias", torch.zeros(n))
        self.register_buffer("running_mean", torch.zeros(n))
        self.register_buffer("running_var", torch.ones(n))

    def forward(self, x):
        scale = self.weight * self.running_var.rsqrt()
        bias = self.bias - self.running_mean * scale
        scale = scale.reshape(1, -1, 1, 1)
        bias = bias.reshape(1, -1, 1, 1)
        print(scale.shape,bias.shape)
        return x * scale + bias

a=FrozenBatchNorm2d((1,2))
a(torch.tensor([1,2,3]))

The running result is different from what I thought.
So can someone tell me what this function exactly does?
I will appreciate it if someone could help me.

Output: "register_buffer" means open an RAM for some parameters which couldn't be optimized or changed during the tranning process, in another word,  the "weight","bias","running_mean","running_var" are consistent values. Hence, that is the reason why this rebuild batchnorm method could be called FrozenBatchnorm2d. It's my explan, hope it can help you.


Instruction: Remove downloaded tensorflow and pytorch(Hugging face) models

Input:
I would like to remove tensorflow and hugging face models from my laptop.
I did find one link https://github.com/huggingface/transformers/issues/861
but is there not command that can remove them because as mentioned in the link manually deleting can cause problems because we don't know which other files are linked to those models or are expecting some model to be present in that location or simply it may cause some error.

Output:
The transformers library will store the downloaded files in your cache. As far as I know, there is no built-in method to remove certain models from the cache. But you can code something by yourself. The files are stored with a cryptical name alongside two additional files that have .json (.h5.json in case of Tensorflow models) and .lock appended to the cryptical name. The json file contains some metadata that can be used to identify the file. The following is an example of such a file:
{"url": "https://cdn.huggingface.co/roberta-base-pytorch_model.bin", "etag": "\"8a60a65d5096de71f572516af7f5a0c4-30\""}

We can now use this information to create a list of your cached files as shown below:
import glob
import json
import re
from collections import OrderedDict
from transformers import TRANSFORMERS_CACHE

metaFiles = glob.glob(TRANSFORMERS_CACHE + '/*.json')
modelRegex = "huggingface\.co\/(.*)(pytorch_model\.bin$|resolve\/main\/tf_model\.h5$)"

cachedModels = {}
cachedTokenizers = {}
for file in metaFiles:
     with open(file) as j:
         data = json.load(j)
         isM = re.search(modelRegex, data['url'])
         if isM:
             cachedModels[isM.group(1)[:-1]] = file
         else:
             cachedTokenizers[data['url'].partition('huggingface.co/')[2]] = file

cachedTokenizers = OrderedDict(sorted(cachedTokenizers.items(), key=lambda k: k[0]))

Now all you have to do is to check the keys of cachedModels and cachedTokenizers and decide if you want to keep them or not. In case you want to delete them, just check for the value of the dictionary and delete the file from the cache. Don't forget to also delete the corresponding *.json and *.lock files.


Instruction: How to get indices of top-K values from a numpy array

Input:
Let suppose I have probabilities from a Pytorch or Keras predictions and result is with the softmax function
from scipy.special import softmax
probs = softmax(np.random.randn(20,10),1) # 20 instances and 10 class probabilities
probs

I want to find top-5 indices from this numpy array. All I want to do is to run a loop on the results something like:
for index in top_5_indices:
    if index in result:
        print('Found')

I'll get if my results are in top-5 results.
Pytorch has top-k function and I have seen numpy.argpartition but I have no idea how to get this done?

Output:
A little more expensive, but argsort would do:
idx = np.argsort(probs, axis=1)[:,-5:]

If we are talking about pytorch:
probs = torch.from_numpy(softmax(np.random.randn(20,10),1))

values, idx = torch.topk(probs, k=5, axis=-1)


Instruction: Incompletable PyTorch with any CUDA version (module 'torch' has no attribute 'cuda')

Input:
I have NVidia 1080TI, Ubuntu x64, and Python 3.6.9 installed.
I was trying to launch PyTorch with command
import torch
print(torch.cuda.is_available)

and expected to see 'True' but met the error:
AttributeError: module 'torch' has no attribute 'cuda'

I tried to update PyTorch and install the last version 1.7.0 with CUDA 11.0 support. After that, I noticed some version discrepancies. nvidia-smi shows CUDA version 11.0 but nvcc -V shows 9.1. Also, I used cat /usr/local/cuda/version.txt to check CUDA version but got the error: cat: /usr/local/cuda/version.txt: No such file or directory
I installed CUDA driver 450.33 after fully nvidia purging but the error remains and nvcc -V still shows 9.1 version (after reboot also).
One more option I addressed to is conda installation but it didn't help.
What I can do to resolve the problem?

Output:
As a result, I had a file named torch.py in my home directory. After the renaming problem was solved.
Thanks. Maybe my answer will be helpful to someone.


Instruction: Debugging pytorch code in pycharm (Feasibility)

Input:
I am trying to run a code in written in python (pytorch code) which when passed as an arguments options trains the Neural network.
if __name__ == "__main__":
    args = docopt(__doc__)
    myparams = args["options"]
    ....
    /* do work */

Now if we have to run this code, I need to call it from console. python3 train.py --option1 123 etc. But in that case the debug points won't work in pycharm. Can anybody clarify how to debug in this scenario? (If you know the way it would be great if you let me know).

Output: Look at menu bar,Run->Edit Configurations->(Chose One Configuration)Parameters


Instruction: PyTorch error in trying to backward through the graph a second time

Input:
I'm trying to run this code: https://github.com/aitorzip/PyTorch-CycleGAN
I modified only the dataloader and transforms to be compatible with my data.
When trying to run it I get this error:

Traceback (most recent call last):
File "models/CycleGANs/train",
line 150, in
loss_D_A.backward()
File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward(
RuntimeError: Trying to backward through the graph a second time, but the saved intermediate
results have already been freed. Specify retain_graph=True when
calling backward the first time.

This is the train loop up to the point of error:
for epoch in range(opt.epoch, opt.n_epochs):
for i, batch in enumerate(dataloader):
    # Set model input
    real_A = Variable(input_A.copy_(batch['A']))
    real_B = Variable(input_B.copy_(batch['B']))

    ##### Generators A2B and B2A #####
    optimizer_G.zero_grad()

    # Identity loss
    # G_A2B(B) should equal B if real B is fed
    same_B = netG_A2B(real_B)
    loss_identity_B = criterion_identity(same_B, real_B)*5.0
    # G_B2A(A) should equal A if real A is fed
    same_A = netG_B2A(real_A)
    loss_identity_A = criterion_identity(same_A, real_A)*5.0

    # GAN loss
    fake_B = netG_A2B(real_A)
    pred_fake = netD_B(fake_B)
    loss_GAN_A2B = criterion_GAN(pred_fake, target_real)

    fake_A = netG_B2A(real_B)
    pred_fake = netD_A(fake_A)
    loss_GAN_B2A = criterion_GAN(pred_fake, target_real)

    # Cycle loss
    # TODO: cycle loss doesn't allow for multimodality. I leave it for now but needs to be thrown out later
    recovered_A = netG_B2A(fake_B)
    loss_cycle_ABA = criterion_cycle(recovered_A, real_A)*10.0

    recovered_B = netG_A2B(fake_A)
    loss_cycle_BAB = criterion_cycle(recovered_B, real_B)*10.0

    # Total loss
    loss_G = loss_identity_A + loss_identity_B + loss_GAN_A2B + loss_GAN_B2A + loss_cycle_ABA + loss_cycle_BAB
    loss_G.backward()

    optimizer_G.step()

    ##### Discriminator A #####
    optimizer_D_A.zero_grad()

    # Real loss
    pred_real = netD_A(real_A)
    loss_D_real = criterion_GAN(pred_real, target_real)

    # Fake loss
    fake_A = fake_A_buffer.push_and_pop(fake_A)
    pred_fale = netD_A(fake_A.detach())
    loss_D_fake = criterion_GAN(pred_fake, target_fake)

    # Total loss
    loss_D_A = (loss_D_real + loss_D_fake)*0.5
    loss_D_A.backward()

I am not familiar at all what it means. My guess is it's something to do with fake_A_buffer. It's just a fake_A_buffer = ReplayBuffer()
class ReplayBuffer():
def __init__(self, max_size=50):
    assert (max_size > 0), 'Empty buffer or trying to create a black hole. Be careful.'
    self.max_size = max_size
    self.data = []

def push_and_pop(self, data):
    to_return = []
    for element in data.data:
        element = torch.unsqueeze(element, 0)
        if len(self.data) < self.max_size:
            self.data.append(element)
            to_return.append(element)
        else:
            if random.uniform(0,1) > 0.5:
                i = random.randint(0, self.max_size-1)
                to_return.append(self.data[i].clone())
                self.data[i] = element
            else:
                to_return.append(element)
    return Variable(torch.cat(to_return))

Error after setting `loss_G.backward(retain_graph=True)

Traceback (most recent call last):   File "models/CycleGANs/train",
line 150, in
loss_D_A.backward()   File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)   File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward( RuntimeError: one of the variables needed for gradient computation has been modified by an
inplace operation: [torch.FloatTensor [3, 64, 7, 7]] is at version 2;
expected version 1 instead. Hint: enable anomaly detection to find the
operation that failed to compute its gradient, with
torch.autograd.set_detect_anomaly(True).

And after setting torch.autograd.set_detect_anomaly(True)

/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py:130:
UserWarning: Error detected in MkldnnConvolutionBackward. Traceback of
forward call that caused the error:
File "models/CycleGANs/train",
line 115, in
fake_B = netG_A2B(real_A)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/Histology-Style-Transfer-Research/models/CycleGANs/models.py",
line 67, in forward
return self.model(x)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py",
line 117, in forward
input = module(input)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/Histology-Style-Transfer-Research/models/CycleGANs/models.py",
line 19, in forward
return x + self.conv_block(x)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/container.py",
line 117, in forward
input = module(input)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py",
line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py",
line 423, in forward
return self._conv_forward(input, self.weight)
File "/opt/conda/lib/python3.8/site-packages/torch/nn/modules/conv.py",
line 419, in _conv_forward
return F.conv2d(input, weight, self.bias, self.stride,  (Triggered internally at
/opt/conda/conda-bld/pytorch_1603729096996/work/torch/csrc/autograd/python_anomaly_mode.cpp:104.)
Variable._execution_engine.run_backward(
Traceback (most recent call
last):   File "models/CycleGANs/train", line 133, in
loss_G.backward(retain_graph=True)
File "/opt/conda/lib/python3.8/site-packages/torch/tensor.py", line 221, in
backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File
"/opt/conda/lib/python3.8/site-packages/torch/autograd/init.py",
line 130, in backward
Variable._execution_engine.run_backward( RuntimeError: Function 'MkldnnConvolutionBackward' returned nan values in its 2th output.

Output: loss_G.backward() should be  loss_G.backward(retain_graph=True) this is because when you use backward normally it doesn't record the operations it performs in the backward pass, retain_graph=True is telling to do so.


Instruction: pytorch: how to apply function over all cells of 4-d tensor

Input:
I'm trying to apply a function over a 4-D tensor (I think about it as a 2-D matrix with a 2-D matrix in each cell) with the following dimensions: [N x N x N x N].
The apply function returns [1 x N] tensor, so after the apply function I'm expecting a tensor of the following dimensions: [N x N x 1 x N].
Example: let's define [4 x 4 x 4 x 4] tensor:
tensor_4d = torch.tensor([[[[1., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[1., 0., 0., 0.],
  [1., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[1., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[1., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]]],
[[[0., 0., 0., 0.],
  [0., 1., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 1., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 1., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 1., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.]]],
[[[0., 0., 1., 0.],
  [0., 0., 0., 0.],
  [0., 0., 1., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 0., 1., 0.],
  [0., 0., 1., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 1., 0.],
  [0., 0., 0., 0.]],
 [[0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 1., 0.],
  [0., 0., 0., 0.]]],
[[[0., 0., 1., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 1.],
  [0., 0., 0., 1.]],
 [[0., 0., 0., 0.],
  [0., 0., 1., 0.],
  [0., 0., 0., 1.],
  [0., 0., 0., 1.]],
 [[0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 1.],
  [0., 0., 0., 1.]],
 [[0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 0.],
  [0., 0., 0., 1.]]]], dtype=torch.float64)

lets look on tensor_4d at [3][0]:
tensor_4d[3][0]
tensor([[0., 0., 1., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 1.],
        [0., 0., 0., 1.]], dtype=torch.float64)

this my apply function :
def apply_function(tensor_2d):
    eigenvalues, eigenvectors = torch.eig(input=tensor_2d, eigenvectors=True)
    return eigenvector[:, 2]

and this the result of the apply function:
apply_function(tensor_4d[3][0])
tensor([-1.0000e+00,  0.0000e+00, 4.0083e-292,  0.0000e+00],
       dtype=torch.float64)

so the apply_function works for each cell.
Next, I'm trying to use the apply_function with the whole matrix, and expecting each cell will contain the result of activating 'apply_function' for this cell. but, when using the apply function I'm getting the following error:
apply_function(tensor_4d)

Traceback (most recent call last):
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2019.2.3\helpers\pydev\_pydevd_bundle\pydevd_exec2.py", line 3, in Exec
    exec(exp, global_vars, local_vars)
  File "<input>", line 1, in <module>
  File "C:/Users/LiavB/PycharmProjects/RBC/Components/RBC_torch.py", line 41, in apply_function
    eigenvalues, eigenvectors = torch.eig(input=tensor_2d, eigenvectors=True)
RuntimeError: invalid argument 1: A should be 2 dimensional at ..\aten\src\TH/generic/THTensorLapack.cpp:206

Output:
Let's try:
new_shape=(-1,)+tensor_4d.shape[2:]

out = (torch.stack([apply_function(t) for t in tensor_4d.view(new_shape)], axis=-1)
      .reshape(new_shape)
)


Instruction: What could be reason of "ValueError: axes don't match array error" for Pytorch U-net segmentation model?

Input:
I'm trying to implement a segmentation model (which i used for another dataset succesfully before) for kaggle dataset called "Carvana Image Masking Challange".
I searched a lot, but still could not figured out what is the reason i am getting this error. There were some suggestion to check image dimension which could be grayscale format but it seems i have 3 channel for both original and mask images.I am grateful for all your support
My code is following:
Libraries
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
from PIL import Image
import numpy as np
import cv2
import matplotlib.pyplot as plt
from torch.utils.data import DataLoader
from torch.utils.data import Dataset as BaseDataset
import albumentations as albu
import torch
import numpy as np
import segmentation_models_pytorch as smp

Data path
DATA_DIR = 'D:/Users/eugur/Belgeler/Jupyter/Segmentation_Kaggle'

x_train_dir = os.path.join(DATA_DIR, 'train')
y_train_dir = os.path.join(DATA_DIR, 'train_masks')

x_valid_dir = os.path.join(DATA_DIR, 'valid')
y_valid_dir = os.path.join(DATA_DIR, 'valid_masks')

x_test_dir = os.path.join(DATA_DIR, 'test')

helper function for data visualization
def visualize(**images):
    """PLot images in one row."""
    n = len(images)
    plt.figure(figsize=(16, 5))
    for i, (name, image) in enumerate(images.items()):
        plt.subplot(1, n, i + 1)
        plt.xticks([])
        plt.yticks([])
        plt.title(' '.join(name.split('_')).title())
        plt.imshow(image)
    plt.show()

Dataset Class
class Dataset(BaseDataset):
    """

    Args:
        images_dir (str): path to images folder
        masks_dir (str): path to segmentation masks folder
        class_values (list): values of classes to extract from segmentation mask
        augmentation (albumentations.Compose): data transfromation pipeline
            (e.g. flip, scale, etc.)
        preprocessing (albumentations.Compose): data preprocessing
            (e.g. noralization, shape manipulation, etc.)

    """

    CLASSES = ['car']

    def __init__(
            self,
            images_dir,
            masks_dir,
            classes=None,
            augmentation=None,
            preprocessing=None,
    ):
        self.ids = os.listdir(images_dir)
        self.images_fps = [os.path.join(images_dir, image_id) for image_id in self.ids]
        self.masks_fps = [os.path.join(masks_dir, image_id.split('.')[0]+'_mask.gif') for image_id in self.ids]

        # convert str names to class values on masks
        self.class_values = [self.CLASSES.index(cls.lower()) for cls in classes]

        self.augmentation = augmentation
        self.preprocessing = preprocessing

    def __getitem__(self, i):

        # read data
        image = cv2.imread(self.images_fps[i])
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
#         mask = cv2.imread(self.masks_fps[i], 0)
        mask = cv2.VideoCapture(self.masks_fps[i],0)
        ret,mask = mask.read()
        mask = mask/255

        # extract certain classes from mask (e.g. cars)
        masks = [(mask == v) for v in self.class_values]
        mask = np.stack(masks, axis=-1).astype('float')

        # apply augmentations
        if self.augmentation:
            sample = self.augmentation(image=image, mask=mask)
            image, mask = sample['image'], sample['mask']

        # apply preprocessing
        if self.preprocessing:
            sample = self.preprocessing(image=image, mask=mask)
            image, mask = sample['image'], sample['mask']

        return image, np.squeeze(mask,axis=3)

    def __len__(self):
        return len(self.ids)

Preprocessing and Augmentation
def get_training_augmentation():
    train_transform = [

        albu.HorizontalFlip(p=0.5),

        albu.ShiftScaleRotate(scale_limit=0.5, rotate_limit=0, shift_limit=0.1, p=1, border_mode=0),

        albu.PadIfNeeded(min_height=320, min_width=320, always_apply=True, border_mode=0),
        albu.RandomCrop(height=320, width=320, always_apply=True),

        albu.IAAAdditiveGaussianNoise(p=0.2),
        albu.IAAPerspective(p=0.5),

        albu.OneOf(
            [
                albu.CLAHE(p=1),
                albu.RandomBrightness(p=1),
                albu.RandomGamma(p=1),
            ],
            p=0.9,
        ),

        albu.OneOf(
            [
                albu.IAASharpen(p=1),
                albu.Blur(blur_limit=3, p=1),
                albu.MotionBlur(blur_limit=3, p=1),
            ],
            p=0.9,
        ),

        albu.OneOf(
            [
                albu.RandomContrast(p=1),
                albu.HueSaturationValue(p=1),
            ],
            p=0.9,
        ),
    ]
    return albu.Compose(train_transform)

def get_validation_augmentation():
    """Add paddings to make image shape divisible by 32"""
    test_transform = [
        albu.PadIfNeeded(384, 480)
    ]
    return albu.Compose(test_transform)

def to_tensor(x, **kwargs):

    return x.transpose(0,2,1).astype('float32')

def get_preprocessing(preprocessing_fn):
    """Construct preprocessing transform

    Args:
        preprocessing_fn (callbale): data normalization function
            (can be specific for each pretrained neural network)
    Return:
        transform: albumentations.Compose

    """

    _transform = [
        albu.Lambda(image=preprocessing_fn),
        albu.Lambda(image=to_tensor, mask=to_tensor),
    ]
    return albu.Compose(_transform)

Model Definition
ENCODER = 'se_resnext50_32x4d'
ENCODER_WEIGHTS = 'imagenet'
CLASSES = ['car']
ACTIVATION = 'sigmoid' # could be None for logits or 'softmax2d' for multicalss segmentation
DEVICE = 'cuda'

# create segmentation model with pretrained encoder
model = smp.FPN(
    encoder_name=ENCODER,
    encoder_weights=ENCODER_WEIGHTS,
    classes=len(CLASSES),
    in_channels=3,
    activation=ACTIVATION,
)

preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)

Data Loader
train_dataset = Dataset(
    x_train_dir,
    y_train_dir,
    preprocessing=get_preprocessing(preprocessing_fn),
    classes=CLASSES,
)

valid_dataset = Dataset(
    x_valid_dir,
    y_valid_dir,
    preprocessing=get_preprocessing(preprocessing_fn),
    classes=CLASSES,
)

train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=0)
valid_loader = DataLoader(valid_dataset, batch_size=1, shuffle=False, num_workers=0)

Optimer Definition
loss = smp.utils.losses.DiceLoss()
metrics = [
    smp.utils.metrics.IoU(threshold=0.5),
]

optimizer = torch.optim.Adam([
    dict(params=model.parameters(), lr=0.0001),
])

Training
train_epoch = smp.utils.train.TrainEpoch(
    model,
    loss=loss,
    metrics=metrics,
    optimizer=optimizer,
    device=DEVICE,
    verbose=True,
)

valid_epoch = smp.utils.train.ValidEpoch(
    model,
    loss=loss,
    metrics=metrics,
    device=DEVICE,
    verbose=True,
)

max_score = 0

for i in range(0, 20):

    print('\nEpoch: {}'.format(i))
    train_logs = train_epoch.run(train_loader)
    valid_logs = valid_epoch.run(valid_loader)

    # do something (save model, change lr, etc.)
    if max_score < valid_logs['iou_score']:
        max_score = valid_logs['iou_score']
        torch.save(model, './best_model.pth')
        print('Model saved!')

    if i == 25:
        optimizer.param_groups[0]['lr'] = 1e-5
        print('Decrease decoder learning rate to 1e-5!')

Error
> Epoch: 0 train:   0%|          | 0/510 [00:00<?, ?it/s]
>
> --------------------------------------------------------------------------- ValueError                                Traceback (most recent call
> last) <ipython-input-208-d2306c5ca0ea> in <module>
>       6
>       7     print('\nEpoch: {}'.format(i))
> ----> 8     train_logs = train_epoch.run(train_loader)
>       9     valid_logs = valid_epoch.run(valid_loader)
>      10
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\segmentation_models_pytorch\utils\train.py
> in run(self, dataloader)
>      43
>      44         with tqdm(dataloader, desc=self.stage_name, file=sys.stdout, disable=not (self.verbose)) as iterator:
> ---> 45             for x, y in iterator:
>      46                 x, y = x.to(self.device), y.to(self.device)
>      47                 loss, y_pred = self.batch_update(x, y)
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\tqdm\std.py
> in __iter__(self)    1169     1170         try:
> -> 1171             for obj in iterable:    1172                 yield obj    1173                 # Update and possibly print the
> progressbar.
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\torch\utils\data\dataloader.py
> in __next__(self)
>     433         if self._sampler_iter is None:
>     434             self._reset()
> --> 435         data = self._next_data()
>     436         self._num_yielded += 1
>     437         if self._dataset_kind == _DatasetKind.Iterable and \
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\torch\utils\data\dataloader.py
> in _next_data(self)
>     473     def _next_data(self):
>     474         index = self._next_index()  # may raise StopIteration
> --> 475         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
>     476         if self._pin_memory:
>     477             data = _utils.pin_memory.pin_memory(data)
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\torch\utils\data\_utils\fetch.py
> in fetch(self, possibly_batched_index)
>      42     def fetch(self, possibly_batched_index):
>      43         if self.auto_collation:
> ---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
>      45         else:
>      46             data = self.dataset[possibly_batched_index]
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\torch\utils\data\_utils\fetch.py
> in <listcomp>(.0)
>      42     def fetch(self, possibly_batched_index):
>      43         if self.auto_collation:
> ---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
>      45         else:
>      46             data = self.dataset[possibly_batched_index]
>
> <ipython-input-146-65256f8f536d> in __getitem__(self, i)
>      54         # apply preprocessing
>      55         if self.preprocessing:
> ---> 56             sample = self.preprocessing(image=image, mask=mask)
>      57             image, mask = sample['image'], sample['mask']
>      58
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\albumentations\core\composition.py
> in __call__(self, force_apply, *args, **data)
>     180                     p.preprocess(data)
>     181
> --> 182             data = t(force_apply=force_apply, **data)
>     183
>     184             if dual_start_end is not None and idx == dual_start_end[1]:
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\albumentations\core\transforms_interface.py
> in __call__(self, force_apply, *args, **kwargs)
>      87                     )
>      88                 kwargs[self.save_key][id(self)] = deepcopy(params)
> ---> 89             return self.apply_with_params(params, **kwargs)
>      90
>      91         return kwargs
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\albumentations\core\transforms_interface.py
> in apply_with_params(self, params, force_apply, **kwargs)
>     100                 target_function = self._get_target_function(key)
>     101                 target_dependencies = {k: kwargs[k] for k in self.target_dependence.get(key, [])}
> --> 102                 res[key] = target_function(arg, **dict(params, **target_dependencies))
>     103             else:
>     104                 res[key] = None
>
> C:\ProgramData\Anaconda3\envs\segmentation\lib\site-packages\albumentations\augmentations\transforms.py
> in apply_to_mask(self, mask, **params)    3068     def
> apply_to_mask(self, mask, **params):    3069         fn =
> self.custom_apply_fns["mask"]
> -> 3070         return fn(mask, **params)    3071     3072     def apply_to_bbox(self, bbox, **params):
>
> <ipython-input-186-4f194a842931> in to_tensor(x, **kwargs)
>      52
>      53
> ---> 54     return x.transpose(0,2,1).astype('float32')
>      55
>      56
>
> ValueError: axes don't match array

Output:
There were 2 problem on the above code;

Mask image size was wrong, expected as (x,y,1) but it was (x,y,3)

Model expect equal size of rows and columns.

After above changes code works well properly.


Instruction: Validation loss is neither increasing or decreasing

Input: Usually when a model overfits, validation loss goes up and training loss goes down from the point of overfitting. But for my case, training loss still goes down but validation loss stays at same level. Hence validation accuracy also stays at same level but training accuracy goes up. I am trying to reconstruct a 2D image from a 3D volume using UNet. Same is the behavior when I am trying to reconstruct 3D volume from 2D image but at higher loss and lower accuracy. Can someone explain the curve that why validation loss is not going down from the point of overfitting?

Output:
The trends show that your model is overfitting. Ways to overcome overfitting include:

Use data augmentation
Use more data
Use Dropout
Use regularization
Try slowing down your learning rate!


Instruction: Pytorch multiprocessing with shared memory causes matmul to be 30x slower (with only two processes)

Input:
I am trying to improve the speed of my reinforcement learning algorithm by using multiprocessing to have multiple workers generating experience at the same time.  Each process just runs the forward pass of my neural net, no gradient computation is needed.
As I understand it, when passing Tensors and nn.Modules across process boundaries (using torch.multiprocessing.Queue or torch.multiprocessing.Pool), the tensor data is moved to shared memory, which shouldn't be any slower than non-shared memory.
However, when I run my multiprocess code with 2 processes (on an 8 core machine), I find that my pytorch operations become more than 30x slower, more than counteracting the speedup from running two processes simultaneously.
I profiled my application to find which operations specifically are slowing down.  I found that much of my time was spend in nn.functional.linear(), specifically on this line inside a Tensor.matmul call:
    output = input.matmul(weight.t())

I added a timer just to this specific matmul call, and I found that when one process is running, this operation takes less than 0.3 milliseconds, but when two processes are running, it takes more than 10 milliseconds. Note that in both cases the weight matrix has been put in shared memory and passed across process boundaries to a worker process, the only difference is that in the second case there are two worker processes instead of one.
For reference, the shapes of input and weight tensors are torch.Size([1, 24, 180]) and torch.Size([31, 180]), respectively.
What could be causing this drastic slowdown?  is there some subtlety to using torch multiprocessing or shared memory that is not mentioned in any of the documentation?  I feel like there must be some hidden lock that is causing contention here, because this drastic slowdown makes no sense to me.

Output: It seems like this was caused by a bad interaction of OpenMP (used by pytorch by default) and multiprocessing.  This is a known issue in pytorch (https://github.com/pytorch/pytorch/issues/17199) and I was even hitting deadlocks in certain configurations I used to debug.  Turning off OpenMP using torch.set_num_threads(1) fixed the issue for me, and immediately sped up my tensor operations in the multiple processes case, presumably, by bypassing internal locking OpenMP was doing.


Instruction: Pytorch - (Categorical) Cross Entropy Loss using one hot encoding and softmax

Input:
I'm looking for a cross entropy loss function in Pytorch that is like the CategoricalCrossEntropyLoss in Tensorflow.
My labels are one hot encoded and the predictions are the outputs of a softmax layer. For example (every sample belongs to one class):
targets = [0, 0, 1]
predictions = [0.1, 0.2, 0.7]

I want to compute the (categorical) cross entropy on the softmax values and do not take the max values of the predictions as a label and then calculate the cross entropy. Unfortunately, I did not find an appropriate solution since Pytorch's CrossEntropyLoss is not what I want and its BCELoss is also not exactly what I need (isn't it?).
Does anyone know which loss function to use in Pytorch or how to deal with it?
Many thanks in advance!

Output:
I thought Tensorflow's CategoricalCrossEntropyLoss was equivalent to PyTorch's CrossEntropyLoss but it seems not. The former takes OHEs while the latter takes labels as well. It seems, however, that the difference is:

torch.nn.CrossEntropyLoss is  a combination of torch.nn.LogSoftmax and torch.nn.NLLLoss():

tf.keras.losses.CategoricalCrossEntropyLoss is something like:

Your predictions have already been through a softmax. So only the negative log-likelihood needs to be applied. Based on what was discussed here, you could try this:
class CategoricalCrossEntropyLoss(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, y_hat, y):
        return F.nll_loss(y_hat.log(), y.argmax(dim=1))

Above the prediction vector is converted from one-hot-encoding to label with torch.Tensor.argmax.

If that's correct why not just use torch.nn.CrossEntropyLoss in the first place? You would just have to remove the softmax on your model's last layer and convert your targets labels.


Instruction: How to call a network in tf.keras.Sequential()?

Input:
If I use pytorch, I could use [index] to loop the layers:
layers = nn.ModuleList()
q = nn.ModuleList()
for _ in range(10):
        layers.append(attn)
        q.append(nn.Linear(dim1, dim2))
list = []
for index, layer in enumerate(self.layers):
        Q = q[index](inputTensor)
        list.append(layer(attn))

so when we use tensorflow, can we still use index like pytorch?
layers = tf.keras.Sequential()
q = tf.keras.Sequential()
for _ in range(10):
        layers.add(attn)
        q.add(Dense(dim2))
list = []
for index, layer in enumerate(layers):
        Q = layer[index](inputTensor)
        list.add(layer(att))

Output:
Yes - it is possible:
model = tf.keras.Sequential([
  tf.keras.layers.Dense(128),
  tf.keras.layers.Dense(1) ])

for layer in model.layers:
        Q = layer


Instruction: How to convert yolov4 weights.wt to pytorch weights .pt?

Input:
I was trying out the yolov4 from https://github.com/theAIGuysCode/YOLOv4-Cloud-Tutorial and I wanted to convert the weights from .wt files to .pt files for pytorch
Is there a way I can do that?

Output:
Pytorch YOLOv4 (I am biased as I am a maintainer) has the ability to do this with darknet2pytorch. The following is an example snippet
from tool.darknet2pytorch import Darknet
WEIGHTS = Darknet(cfgfile)
WEIGHTS.load_weights(weightfile)

Where cfgfile is your darknet config.cfg file, and weightfile is your darknet .wt weights.
WEIGHTS is now an ordinary PyTorch model, which you can save however you'd like.


Instruction: Implementing a simple optimization algorithm in PyTorch

Input:
I'm currently learning PyTorch in order to utilize its open source autograd feature, and as an exercise for myself, I want to implement a simple optimization algorithm that I've already implemented in MATLAB. As a simple example, say I'm trying to solve the problem min_x 1/2 x'Ax - b'x, i.e. find the vector x which minimizes the quantity x'Ax - b'x, given both A and b. A simple gradient descent algorithm with exact line search in MATLAB may look something like this:
% initialize x = zeros(n, 1) where n is the length of b
while residual > tolerance
    grad = A*x - b; % compute the gradient of the objective
    alpha = norm(grad)^2/(grad'*A*grad); % compute step-size alpha by exact line search
    x = x - alpha*grad; % do a gradient step
    residual = norm(grad); % compute residual
    objective = x'*A*x - b'*x; % compute objective value at current iteration

How would I implement this optimization algorithm in PyTorch? Specifically, I would want to do an identical optimization loop, where I replace my own computation of the gradient with Torch's autograd feature. In other words, I want to perform the exact same algorithm as above in PyTorch, except instead of computing the gradient myself, I simply use PyTorch's autograd feature to compute the gradient. In this way, I don't want to call any given optimizer (like SGD or Adam)--just write the algorithm myself with the only difference being that the gradient is computed by PyTorch. I plan to compare the results of the above numpy/MATLAB implementation with the explicit gradient computation vs the PyTorch implementation with what I assume is a numerical approximation of the gradient.
Thank you very much!

Output:
Basically you need to use the autograd in pytorch. Not a complete program but it'll look something like this along the lines:
In each iteration do the following:

Specify x.requires_grad=True because you need its gradient. Then compute your objective function:

x.requires_grad = True
obj_function = torch.matmul(x.t(),torch.matmul(A,x)) * .5 - torch.matmul(b,x)

Clear old gradient results of x:

x.grad = None

Call backward() on obj_function. This will automatically compute the gradients of the tensors involved in the operation:

obj_function.backward()

Set x.requires_grad to false because you need to modify x, but pytorch does not allow inplace modification of a tensor that requires grad:

x.requires_grad = False
x = x - x.grad * step_size


Instruction: dropout(): argument 'input' (position 1) must be Tensor, not str when using Bert with Huggingface

Input:
My code was working fine and when I tried to run it today without changing anything I got the following error:
dropout(): argument 'input' (position 1) must be Tensor, not str

Would appreciate if help could be provided.
Could be an issue with the data loader?

Output:
if you use HuggingFace, this information could be useful. I have same error and fix it with adding parameter return_dict=False in model class before dropout:
outputs = model(**inputs, return_dict=False)


Instruction: PyTorch's DataParallel is only using one GPU

Input:
I'm trying to use two GPU's for training a model in PyTorch. I'm using torch.nn.DataParallel but for some reason nvidia-smi is saying that I'm only using one GPU.
The code is something along the lines of:
>>> import torch.nn as nn
>>> model = SomeModel()
>>> model = nn.DataParallel(model)
>>> model.to('cuda')

When I run the program and observe the output of nvidia-smi, I only see GPU 0 running. Would anybody know what the problem is?

Output: You should be using nn.DataParallel(model, [0,1]) in order to use GPU #0 and GPU #1. The call model.to('cuda') afterwards is not necessary. You may be tempted to use nn.DataParallel(model.to('cuda'), [0,1]), but this appears unnecessary as well.


Instruction: Run Pytorch stacked model on Colab TPU

Input:
I am trying to run this my model on Colab Multi core TPU but I really don't know how to do it. I tried this tutorial notebook but I got some error and I can't fix it but I think there is maybe simpler wait for to do it.
About my model:
class BERTModel(nn.Module):
    def __init__(self,...):
        super().__init__()
        if ...:
            self.bert_model = XLMRobertaModel.from_pretrained(...)   # huggingface XLM-R
        elif ...:
            self.bert_model = others_model.from_pretrained(...)   # huggingface XLM-R

        ... # some other model's parameters

    def forward(self,...):
        bert_input = ...
        output = self.bert_model(bert_input)

        ... # some function that process on output

    def other_function(self,...):
        # just doing some process on output. like concat layers's embedding and return ...

class MAINModel(nn.Module):
    def __init__(self,...):
        super().__init__()

        print('Using model 1')
        self.bert_model_1 = BERTModel(...)

        print('Using model 2')
        self.bert_model_2 = BERTModel(...)

        self.linear = nn.Linear(...)

    def forward(self,...):
        bert_input = ...
        bert_output = self.bert_model(bert_input)
        linear_output = self.linear(bert_output)

        return linear_output

Can you please tell me how to run a model like my model on Colab TPU? I used Colab PRO to make sure Ram memory is not a big problem. Thanks you so so much.

Output:
I would work off the examples here: https://github.com/pytorch/xla/tree/master/contrib/colab
Maybe start with a simpler model like this: https://github.com/pytorch/xla/blob/master/contrib/colab/mnist-training.ipynb
In the pseudocode you shared, there is no reference to the torch_xla library, which is required to use PyTorch on TPUs. I'd recommend starting with on of the working Colab notebooks in that directory I shared and then swapping out parts of the model with your own model. There are a few (usually like 3-4) places in the overall training code you need to modify for a model that runs on GPUs using native PyTorch if you want to run that model on TPUs. See here for a description of some of the changes. The other big change is to wrap the default dataloader with a ParallelLoader as shown in the example MNIST colab I shared
If you have any specific error you see in one of the Colabs, feel free to open an issue : https://github.com/pytorch/xla/issues


Instruction: Does pytorch have function to calculate correlation coefficient matrix like numpy.corrcoef ()

Input: Does pytorch have function to calculate correlation coefficient matrix like numpy.corrcoef ()

Output: fastai (implemented heavily in pytorch) provides a suite of correlation coefficients including Pearson, Spearman, and Matthews (which probably is not what you want).  fastai's documentation lists all of the stored commands here. You can use them via hooks during Pytorch training.


Instruction: The difference of loading model parameters between load_state_dict and nn.Parameter in pytorch

Input:
When I wanna assign part of pre-trained model parameters to another module defined in a new model of PyTorch, I got two different outputs using two different methods.
The Network is defined as follows:
class Net:
    def __init__(self):
        super(Net, self).__init__()

        self.resnet = torch.hub.load('pytorch/vision', 'resnet18', pretrained=True)
        self.resnet = nn.Sequential(*list(self.resnet.children())[:-1])
        self.freeze_model(self.resnet)
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Linear(512, 256),
            nn.ReLU(),
            nn.Linear(256, 3),
        )

    def forward(self, x):
        out = self.resnet(x)
        out = out.flatten(start_dim=1)
        out = self.classifier(out)
        return out

What I want is to assign pre-trained parameters to classifier in the net module. Two different ways were used for this task.
# First way
net.load_state_dict(torch.load('model_CNN_pretrained.ptl'))

# Second way
params = torch.load('model_CNN_pretrained.ptl')
net.classifier[1].weight = nn.Parameter(params['classifier.1.weight'], requires_grad =False)
net.classifier[1].bias = nn.Parameter(params['classifier.1.bias'], requires_grad =False)
net.classifier[3].weight = nn.Parameter(params['classifier.3.weight'], requires_grad =False)
net.classifier[3].bias = nn.Parameter(params['classifier.3.bias'], requires_grad =False)

The parameters were assigned correctly but got two different outputs from the same input data. The first method works correctly, but the second doesn't work well. Could some guys point what the difference of these two methods?

Output:
Finally, I find out where is the problem.
During the pre-trained process, buffer parameters in BatchNorm2d Layer of ResNet18 model were changed even if we set require_grad of parameters False. Buffer parameters were calculated by the input data after model.train() was processed, and unchanged after model.eval().
There is a link about how to freeze the BN layer.
How to freeze BN layers while training the rest of network (mean and var wont freeze)


Instruction: PyTorch mutiprocessing: Do I need to use Lock() when accessing a shared model?

Input:
I have some questions about using the torch.multiprocessing module. Let’s say I have a torch.nn.Module called model and I call model.share_memory() on it.
What happens if two threads call the forward(), i.e. model(input) at the same time? Is it safe? Or should I use Lock mechanisms to be sure that model is not accessed at the same time by multiple threads?
Similarly, what happens if two or more threads have an optimizer working on model.parameters() and they call optimizer.step() at the same time?
I ask these questions because I often see the optimizer.step() being called on shared models without lock mechanisms (i.e. in RL implementations of A3C or ACER) and I wonder if it is a safe thing to do.

Output:
It doesn't have to be safe, since they are running asynchronously not in parallel. Quoting from the docs,

Using torch.multiprocessing, it is possible to train a model
asynchronously, with parameters either shared all the time, or being
periodically synchronized. In the first case, we recommend sending
over the whole model object, while in the latter, we advise to only
send the state_dict().


Instruction: PyTorch's torch.nn.functional.interpolate behaving strangely

Input:
I'm having issues with PyTorch's tensor-resizing options. In the following code, x is a dataset of 888 64x64 RGB images of pokemon. xs is to be a dictionary of the same dataset at different resolutions.
def load_data():
    pokemon = []
    for png in os.listdir("pokemon"):
        pokemon.append(imageio.imread("pokemon/" + png))
    pokemon = np.array(pokemon)
    s = pokemon.shape
    pokemon = pokemon.reshape((s[0], s[-1], s[1], s[2]))
    pokemon = (pokemon.astype(np.float32) - 127.5)/127.5
    return(pokemon)
x = load_data()

ss = [2, 1, 1/2, 1/4, 1/8, 1/16]
xs = {s : nn.functional.interpolate(torch.from_numpy(x), scale_factor = s, mode='nearest') for s in ss}

As expected, the output shapes include all 888 RGB images with new heights and widths.
print("Data shape:")
for x in xs.values():
    print(" ", x.shape)

> Data shape:
>  torch.Size([888, 3, 128, 128])
>  torch.Size([888, 3, 64, 64])
>  torch.Size([888, 3, 32, 32])
>  torch.Size([888, 3, 16, 16])
>  torch.Size([888, 3, 8, 8])
>  torch.Size([888, 3, 4, 4])

However, the images weren't sensibly resized.

When s = 1, the pokemon are in their ordinary 64x64 format, as expected:

When s = 2, the pokemon become Pollocks, overlapping in different colors:

When s = 1/2, the pokemon are cut into pieces and rearranged:

Is there an alternative PyTorch tool for resizing? I encounter roughly the same problem using AvgPool2d and Upsample instead. I will be resizing non-image tensors, too, so PIL isn't always an option. In any case, using the GPU instead of CPU would be helpful.

Output: I have solved my problem. I was unused to transitioning images from (Batch, Height, Width, Channels) to (Batch, Channels, Height, Width) for PyTorch. Replacing the reshape lines with np.moveaxis fixed the issue. Thanks, everyone, for your help.


Instruction: Should a data batch be moved to CPU and converted (from torch Tensor) to a numpy array when doing evaluation w.r.t. a metric during the training?

Input:
I am going through Andrew Ng’s tutorial from the CS230 Stanford course, and in every epoch of the training, evaluation is performed by calculating the metrics.
But before calculating the metrics, they are sending the batches to CPU and converting them to numpy arrays (code here).
# extract data from torch Variable, move to cpu, convert to numpy arrays
output_batch = output_batch.data.cpu().numpy()
labels_batch = labels_batch.data.cpu().numpy()

# compute all metrics on this batch
summary_batch = {metric: metrics[metric](output_batch, labels_batch) for metric in metrics}

My question is: why do they do that? Why don’t they just calculate the metrics (which is done here) on GPU using torch methods (e.g. torch.sum as opposed to np.sum)?
I would think that GPU to CPU transfers would slow things down, so there should be a very good reason for doing them?
I am new to PyTorch so I might be missing something.

Output: Correct me if I'm wrong. Sending back the data to the CPU allows to reduce the GPU load even though memory is replaced when entering the following loop cycle. Futhermore, I believe converting to numpy has the advantage of freeing memory since you are detaching your data from the calculation graph. You end up manipulating labels_batch.cpu().numpy() a fixed array vs labels_batch a tensor attached to the entire network through linked backward_fn callbacks.


Instruction: How to Create Class Label for Mosaic Augmentation in Image Classification?

Input:
To create a class label in CutMix or MixUp type augmentation, we can use beta such as np.random.beta or scipy.stats.beta and do as follows for two labels:
label = label_one*beta + (1-beta)*label_two

But what if we've more than two images? In YoLo4, they've tried an interesting augmentation called Mosaic Augmentation for object detection problems. Unlike CutMix or MixUp, this augmentation creates augmented samples with 4 images. In object detection cases, we can compute the shift of each instance co-ords and thus possible to get the proper ground truth, here. But for only image classification cases, how can we do that?
Here is a starter.
import tensorflow as tf
import matplotlib.pyplot as plt
import random

(train_images, train_labels), (test_images, test_labels) = \
tf.keras.datasets.cifar10.load_data()
train_images = train_images[:10,:,:]
train_labels = train_labels[:10]
train_images.shape, train_labels.shape

((10, 32, 32, 3), (10, 1))

Here is a function we've written for this augmentation; ( too ugly with an `inner-outer loop! Please suggest if we can do it efficiently.)
def mosaicmix(image, label, DIM, minfrac=0.25, maxfrac=0.75):
    '''image, label: batches of samples
    '''
    xc, yc  = np.random.randint(DIM * minfrac, DIM * maxfrac, (2,))
    indices = np.random.permutation(int(image.shape[0]))
    mosaic_image = np.zeros((DIM, DIM, 3), dtype=np.float32)
    final_imgs, final_lbs = [], []

    # Iterate over the full indices
    for j in range(len(indices)):
        # Take 4 sample for to create a mosaic sample randomly
        rand4indices = [j] + random.sample(list(indices), 3)

        # Make mosaic with 4 samples
        for i in range(len(rand4indices)):
            if i == 0:    # top left
                x1a, y1a, x2a, y2a =  0,  0, xc, yc
                x1b, y1b, x2b, y2b = DIM - xc, DIM - yc, DIM, DIM # from bottom right
            elif i == 1:  # top right
                x1a, y1a, x2a, y2a = xc, 0, DIM , yc
                x1b, y1b, x2b, y2b = 0, DIM - yc, DIM - xc, DIM # from bottom left
            elif i == 2:  # bottom left
                x1a, y1a, x2a, y2a = 0, yc, xc, DIM
                x1b, y1b, x2b, y2b = DIM - xc, 0, DIM, DIM-yc   # from top right
            elif i == 3:  # bottom right
                x1a, y1a, x2a, y2a = xc, yc,  DIM, DIM
                x1b, y1b, x2b, y2b = 0, 0, DIM-xc, DIM-yc    # from top left

            # Copy-Paste
            mosaic_image[y1a:y2a, x1a:x2a] = image[i,][y1b:y2b, x1b:x2b]

        # Append the Mosiac samples
        final_imgs.append(mosaic_image)

    return final_imgs, label

The augmented samples, currently with the wrong labels.
data, label = mosaicmix(train_images, train_labels, 32)
plt.imshow(data[5]/255)

However, here are some more examples to motivate you. Data is from the Cassava Leaf competition.

(source: googleapis.com)

(source: googleapis.com)

Output:
We already know that, in CutMix, λ is a float number from the beta distribution Beta(α,α). We have seen, when α=1, it performs best. Now, If we grant α==1 always, we can say that λ is sampled from the uniform distribution..
Simply we can say λ is just a floating-point number which value will be 0 to 1.
So, only for 2 images,
if we use λ for the 1st image then we can calculate the remaining unknown portion simply by 1-λ.
But for 3 images, if we use λ for the 1st image, we cannot calculate other 2 unknowns from that single λ.  If we really want to do so, we need 2 random numbers for 3 images. In the same way, we can say that for the n number of images, we need the n-1 number random variable. And in all cases, the summation should be 1. (for example, λ + (1-λ) == 1). If the sum is not 1, the label will be wrong!
For this purpose  Dirichlet distribution might be helpful because it helps to generate quantities that sum to 1. A Dirichlet-distributed random variable can be seen as a multivariate generalization of a Beta distribution.
>>> np.random.dirichlet((1, 1), 1)  # for 2 images. Equivalent to λ and (1-λ)
array([[0.92870347, 0.07129653]])
>>> np.random.dirichlet((1, 1, 1), 1)  # for 3 images.
array([[0.38712673, 0.46132787, 0.1515454 ]])
>>> np.random.dirichlet((1, 1, 1, 1), 1)  # for 4 images.
array([[0.59482542, 0.0185333 , 0.33322484, 0.05341645]])

In CutMix, the size of the cropped part of an image has a relation with λ which weighting the corresponding labels.

So, for multiple λ, you also need to calculate them accordingly.
# let's say for 4 images
# I am not sure the proper way.

image_list = [4 images]
label_list = [4 label]
new_img = np.zeros((w, h))

beta_list = np.random.dirichlet((1, 1, 1, 1), 1)[0]
for idx, beta in enumerate(beta_list):
    x0, y0, w, h = get_cropping_params(beta, full_img)  # something like this
    new_img[x0, y0, w, h] = image_list[idx][x0, y0, w, h]
    label_list[idx] = label_list[idx] * beta


Instruction: How to process TransformerEncoderLayer output in pytorch

Input:
I am trying to use bio-bert sentence embeddings for text classification of longer pieces of text.
As it currently stands I standardize the number of sentences in each piece of text (some sentences are only comprised of ("[PAD]") and run each sentence through biobert to get sentence vectors as they do here:
https://pypi.org/project/biobert-embedding/
I then run those embeddings through a TrasnformerEncoder with 8 layers and 16 attention heads.
The TrasnformerEncoder outputs something of shape (batch_size, num_sentences, embedding_size).
I then try to decode this with a Linear layer and map it to my classes (of which there are 7) and softmax the output to get probabilities.
My loss function is simply nn.CrossEntropyLoss().
At first, I summed over dimensions 1 of the TransformerEncoder output to get something of size (batch_size, embedding_size). This invariable led to my network converging on always predicting one of the labels with absolute certainty. Usually the most common label in the dataset.
I then tried only taking the output for the last sentence of the TransformerEncoder output. i.e. TransformerEncoderOutput[:, -1, :].
This resulted in something similar.
I then tried running my Linear layer on each of the outputs of TransformerEncoder output to produce a tensor of size (batch_size, num_sentences, 7). I then sum over dim 1 (makes a tensor of size (batch_size, 7) and softmax as usual. The idea here is that every sentence gets to vote for the label after being informed about its place in the sequence.
This converged even more quickly to just predicting 1 for one of the labels and vanishingly small values for the others.
I feel like I am misunderstanding out to use the output of a pytorch Transformer somehow.
My learning rate is very low, 0.00001, and that helped delay the convergence but it converged eventually anyway.
What this suggests to me is that my network is incapable of figuring anything out about the text and is just learning to find the most common labels. I would guess that this is either a problem with my loss function or a problem with how I am using the Transformer.
Is there a glaring flaw in the architecture that I have laid out?

Output:
So the input and output shape of the transformer-encoder is batch-size, sequence-length, embedding-size).
There are three possibilities to process the output of the transformer encoder (when not using the decoder).

you take the mean of the sequence-length dimension:

x = self.transformer_encoder(x)
x = x.reshape(batch_size, seq_size, embedding_size)
x = x.mean(1)

sum it up as you said:

x = self.transformer_encoder(x)
x = x.reshape(batch_size, seq_size, embedding_size)
x = x.sum(1)

using a recurrent neural network to combine the information along the sequence-length dimension:

x = self.transformer_encoder(x)
x = x.reshape(batch_size, seq_len, embedding_size)

# init hidden state
hidden = torch.zeros(layers, batch_size, embedding_size).to(device=device)
x, hidden = self.rnn(x, hidden)
x = x.reshape(batch_size, seq_size, embedding_size)

# take last output
x = x[:, -1]

Taking the last element of the Transformer output isnt a good idea I think. Because then you only take 1 / seq-len of the information. But using rnn, the last output has still information of every other output.
Id say that taking the mean is the best idea.
And to the learning rate: For me it always worked very much better when I used warmup training. If you dont know what that is: You start at a low learning rate, for example 0.00001 and you increase it until you have reached some target lr, for example 0.002. And from then you just decay the lr as usual.


Instruction: Diagonal embedding of a batch of matrices in pytorch?

Input:
If you are given a collection of n x n matrices say m of them, is there a predefined function in pytorch that performs a diagonal embedding on all of these into a larger matrix of dimension nm x nm?
To be concrete, what I am looking for is say you have two 2 x 2 identity matrices, then their diagonal embedding into a 4 x 4 matrix would be the identity 4 x 4 matrix.
Something like:
torch.block_diag

but this expects you to feed each matrix as a separate argument.

Output:
Your question doesn't specify how you get your m tensors. Let's say you have
# channel first tensors
a = torch.ones(4,2,2)

or
# a list of tensors
a = [torch.ones(2,2) for _ in range(4)]

then you can unpack that in block_diag:
>>> torch.block_diag(*a)

tensor([[1., 1., 0., 0., 0., 0., 0., 0.],
        [1., 1., 0., 0., 0., 0., 0., 0.],
        [0., 0., 1., 1., 0., 0., 0., 0.],
        [0., 0., 1., 1., 0., 0., 0., 0.],
        [0., 0., 0., 0., 1., 1., 0., 0.],
        [0., 0., 0., 0., 1., 1., 0., 0.],
        [0., 0., 0., 0., 0., 0., 1., 1.],
        [0., 0., 0., 0., 0., 0., 1., 1.]])


Instruction: Convert a simple cnn from keras to pytorch

Input:
Can anyone please help me to convert this model to PyTorch? I already tried to convert from Keras to PyTorch like this How can I convert this keras cnn model to pytorch version but training results were different. Thank you.
input_3d = (1, 64, 96, 96)
pool_3d = (2, 2, 2)
model = Sequential()
model.add(Convolution3D(8, 3, 3, 3, name='conv1', input_shape=input_3d,
                          data_format='channels_first'))
model.add(MaxPooling3D(pool_size=pool_3d, name='pool1'))
model.add(Convolution3D(8, 3, 3, 3, name='conv2',data_format='channels_first'))
model.add(MaxPooling3D(pool_size=pool_3d, name='pool2'))
model.add(Convolution3D(8, 3, 3, 3, name='conv3',data_format='channels_first'))
model.add(MaxPooling3D(pool_size=pool_3d, name='pool3'))
model.add(Flatten())
model.add(Dense(2000, activation='relu', name='dense1'))
model.add(Dropout(0.5, name='dropout1'))
model.add(Dense(500, activation='relu', name='dense2'))
model.add(Dropout(0.5, name='dropout2'))
model.add(Dense(3, activation='softmax', name='softmax'))

_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
conv1 (Conv3D)               (None, 8, 60, 94, 94)     224
_________________________________________________________________
pool1 (MaxPooling3D)         (None, 8, 30, 47, 47)     0
_________________________________________________________________
conv2 (Conv3D)               (None, 8, 28, 45, 45)     1736
_________________________________________________________________
pool2 (MaxPooling3D)         (None, 8, 14, 22, 22)     0
_________________________________________________________________
conv3 (Conv3D)               (None, 8, 12, 20, 20)     1736
_________________________________________________________________
pool3 (MaxPooling3D)         (None, 8, 6, 10, 10)      0
_________________________________________________________________
flatten_1 (Flatten)          (None, 4800)              0
_________________________________________________________________
dense1 (Dense)               (None, 2000)              9602000
_________________________________________________________________
dropout1 (Dropout)           (None, 2000)              0
_________________________________________________________________
dense2 (Dense)               (None, 500)               1000500
_________________________________________________________________
dropout2 (Dropout)           (None, 500)               0
_________________________________________________________________
softmax (Dense)              (None, 3)                 1503
=================================================================

Output:
Your PyTorch equivalent of the Keras model would look like this:
class CNN(nn.Module):

    def __init__(self, ):
        super(CNN, self).__init__()

        self.maxpool = nn.MaxPool3d((2, 2, 2))

        self.conv1 = nn.Conv3d(in_channels=1, out_channels=8, kernel_size=3)
        self.conv2 = nn.Conv3d(in_channels=8, out_channels=8, kernel_size=3)
        self.conv3 = nn.Conv3d(in_channels=8, out_channels=8, kernel_size=3)

        self.linear1 = nn.Linear(4800, 2000)
        self.dropout1 = nn.Dropout3d(0.5)

        self.linear2 = nn.Linear(2000, 500)
        self.dropout2 = nn.Dropout3d(0.5)

        self.linear3 = nn.Linear(500, 3)

    def forward(self, x):

        out = self.maxpool(self.conv1(x))
        out = self.maxpool(self.conv2(out))
        out = self.maxpool(self.conv3(out))

        # Flattening process
        b, c, d, h, w = out.size() # batch_size, channels, depth, height, width
        out = out.view(-1, c * d * h * w)

        out = self.dropout1(self.linear1(out))
        out = self.dropout2(self.linear2(out))
        out = self.linear3(out)

        out = torch.softmax(out, 1)

        return out

A driver program to test the model:
inputs = torch.randn(8, 1, 64, 96, 96)
model = CNN()
outputs = model(inputs)
print(outputs.shape) # torch.Size([8, 3])