Instruction: Why VAE loss doesn’t converge to zero?

Input:
I’m using a Variational Autoencoder and this is my implementation for the loss function:
class VariationalAutoencoder(nn.Module):
    # ...some functions...

    def gaussian_likelihood(self, x_hat, logscale, x):
        scale = torch.exp(logscale)
        mean = x_hat
        dist = torch.distributions.Normal(mean, scale)
        # measure prob of seeing image under p(x|z)
        log_pxz = dist.log_prob(x)
        return log_pxz.sum(dim=(1, 2, 3))

    def forward(self, input):
        mu, logvar = self.encode(input)
        z = self.reparameterise(mu, logvar)
        return self.decoder(z), mu, logvar, z

    def loss_function(self, x_hat, x, mu, logvar, β=1):
        std = torch.exp(logvar / 2)
        q = torch.distributions.Normal(mu, std)
        z = q.rsample()

        # reconstruction loss
        recon_loss = self.gaussian_likelihood(x_hat, self.log_scale, x)

        # kl
        kl = self.kl_divergence(z, mu, std)

        # elbo
        elbo = (kl - recon_loss)
        elbo = elbo.mean()
        return elbo

    def kl_divergence(self, z, mu, std):
        # --------------------------
        # Monte carlo KL divergence
        # --------------------------
        # 1. define the first two probabilities (in this case Normal for both)
        p = torch.distributions.Normal(torch.zeros_like(mu), torch.ones_like(std))
        q = torch.distributions.Normal(mu, std)

        # 2. get the probabilities from the equation
        log_qzx = q.log_prob(z)
        log_pz = p.log_prob(z)

        # kl
        kl = (log_qzx - log_pz)
        kl = kl.sum(-1)

        return kl

I use Sigmoid() function when exiting the decoder. I train the model in this way:
for epoch in range(0, epochs + 1):
        if epoch > 0:  # test untrained net first
            model.train()
            train_loss = 0
            loop = tqdm(train_loader)
            optimizer = model.setOptimizer(model)
            for x in loop:
                x = x.to(device)
                x_hat, mu, logvar, features = model(x)
                loss = model.loss_function(x_hat, x, mu, logvar)
                train_loss += loss.item()
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
                loop.set_postfix(loss=loss)
            train_loss = train_loss /= len(train_loader.dataset)
            print(f'====> Epoch: {epoch} Average loss: {train_loss:.4f}')

The loss doesn’t settle at zero but but becomes negative (around -2). If I remove train_loss = train_loss /= len(train_loader.dataset), it’s diverges completely.
How can I make the loss converge to zero?

Output: Loss for VAE can be negative. It has a log-likelihood - which can be negative. There is nothing wrong in that.


Instruction: How to randomly set a variable number of elements in each row of a tensor in PyTorch

Input:
I want to create a zero-one matrix of dimension (n, n). The ones should be placed randomly, with a cap on the number of ones in each row. Let us say I have a list of length n that has the value of cap for each of the n rows. How can I do this in PyTorch?
My question is similar to this previous question. The only change I am looking for is, there should be n values of k, corresponding to n rows.

Output:
As explained by @Marcel in the comments above, you can first set the first m values to value k then index by permuted indices in order to get a shuffle tensor:
>>> n = 10; m = 3; k = 1
>>> x = torch.zeros(n, n)

>>> x[:, :m] = k
tensor([[1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0.]])

Use torch.randperm to get row-wise column permutations:
>>> perm = torch.stack([torch.randperm(10) for _ in range(len(x))])
tensor([[8, 0, 3, 2, 1, 6, 9, 4, 5, 7],
        [5, 7, 1, 4, 8, 0, 6, 9, 2, 3],
        [2, 1, 9, 7, 0, 8, 6, 3, 5, 4],
        [1, 3, 5, 8, 7, 6, 9, 4, 2, 0],
        [7, 6, 0, 5, 2, 9, 1, 8, 4, 3],
        [5, 0, 6, 8, 1, 9, 2, 4, 3, 7],
        [4, 0, 6, 5, 8, 1, 3, 7, 2, 9],
        [5, 3, 4, 9, 0, 1, 7, 6, 8, 2],
        [5, 7, 9, 3, 2, 6, 8, 0, 4, 1],
        [2, 7, 4, 6, 3, 0, 9, 8, 5, 1]])

Then use torch.gather to index the tensor x with perm:
>>> x.gather(dim=0, index=perm)
tensor([[0., 1., 0., 1., 1., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 1., 0., 0., 1., 0.],
        [1., 1., 0., 0., 1., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0., 1., 1.],
        [0., 0., 1., 0., 1., 0., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1., 0., 1., 0., 0., 0.],
        [0., 1., 0., 0., 0., 1., 0., 0., 1., 0.],
        [0., 0., 0., 0., 1., 1., 0., 0., 0., 1.],
        [0., 0., 0., 0., 1., 0., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0., 1., 0., 0., 0., 1.]])

Alternatively you can use torch.scatter straight way with the value keyword argument:
>>> torch.zeros(n, n).scatter(dim=0, index=perm, value=1)
tensor([[0., 1., 0., 1., 1., 0., 0., 0., 0., 0.],
        [0., 0., 1., 0., 0., 1., 0., 0., 1., 0.],
        [1., 1., 0., 0., 1., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0., 1., 1.],
        [0., 0., 1., 0., 1., 0., 1., 0., 0., 0.],
        [0., 1., 0., 0., 1., 0., 1., 0., 0., 0.],
        [0., 1., 0., 0., 0., 1., 0., 0., 1., 0.],
        [0., 0., 0., 0., 1., 1., 0., 0., 0., 1.],
        [0., 0., 0., 0., 1., 0., 0., 1., 0., 1.],
        [1., 0., 0., 0., 0., 1., 0., 0., 0., 1.]])

If m is a tensor itself, you can find a workaround using a combination of torch.arange and torch.where:
First encode the positions:
>>> d = torch.arange(n)[None].repeat(n,1)
>>> x = torch.where(d+m>n, 0, 1)
tensor([[1, 1, 1, 1, 1, 1, 0, 0, 0, 0],
        [1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 0, 0],
        [1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])

Construct the permutation as before:
>>> perm = torch.stack([torch.randperm(10) for _ in range(n)])
tensor([[2, 5, 7, 0, 4, 1, 3, 6, 8, 9],
        [7, 4, 9, 5, 6, 0, 3, 1, 2, 8],
        [5, 1, 4, 9, 0, 3, 2, 6, 7, 8],
        [9, 6, 0, 2, 3, 1, 7, 5, 4, 8],
        [3, 5, 4, 6, 0, 7, 9, 8, 2, 1],
        [5, 7, 8, 6, 9, 2, 0, 4, 3, 1],
        [8, 3, 9, 0, 6, 2, 5, 7, 4, 1],
        [2, 9, 4, 3, 7, 8, 1, 0, 6, 5],
        [5, 4, 8, 3, 2, 9, 7, 1, 6, 0],
        [8, 7, 3, 6, 5, 4, 2, 0, 9, 1]])

Then scatter on x:
>>> x.scatter(dim=0, index=perm, value=1)
tensor([[1, 1, 1, 1, 1, 1, 1, 1, 0, 1],
        [1, 1, 1, 0, 0, 1, 1, 1, 0, 1],
        [1, 1, 1, 1, 1, 1, 1, 0, 1, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 0, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
        [1, 1, 1, 0, 1, 1, 1, 1, 1, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])


Instruction: Implementation of multitask "nested" neural network

Input:
I am trying to implement a multitask neural network used by a paper but am quite unsure how I should code the multitask network because the authors did not provide code for that part.
The network architecture looks like (paper):

To make it simpler, the network architecture could be generalized as (For demo I changed their more complicated operation for the pair of individual embeddings to concatenation):

The authors are summing the loss from the individual tasks and the pairwise tasks, and using the total loss to optimize the parameters for the three networks (encoder, MLP-1, MLP-2) in each batch, but I am kind of at sea as to how different types of data are combined in a single batch to feed into two different networks that share an initial encoder. I tried to search for other networks with similar structure but did not find any sources. Would appreciate any thoughts!

Output:
This is actually a common pattern. It would be solved by code like the following.
class Network(nn.Module):
   def __init__(self, ...):
      self.encoder = DrugTargetInteractiongNetwork()
      self.mlp1 = ClassificationMLP()
      self.mlp2 = PairwiseMLP()

   def forward(self, data_a, data_b):
      a_encoded = self.encoder(data_a)
      b_encoded = self.encoder(data_b)

      a_classified = self.mlp1(a_encoded)
      b_classified = self.mlp1(b_encoded)

      # let me assume data_a and data_b are of shape
      # [batch_size, n_molecules, n_features].
      # and that those n_molecules are not necessarily
      # equal.
      # This can be generalized to more dimensions.
      a_broadcast, b_broadcast = torch.broadcast_tensors(
         a_encoded[:, None, :, :],
         b_encoded[:, :, None, :],
      )

      # this will work if your mlp2 accepts an arbitrary number of
      # learding dimensions and just broadcasts over them. That's true
      # for example if it uses just Linear and pointwise
      # operations, but may fail if it makes some specific assumptions
      # about the number of dimensions of the inputs
      pairwise_classified = self.mlp2(a_broadcast, b_broadcast)

      # if that is a problem, you have to reshape it such that it
      # works. Most torch models accept at least a leading batch dimension
      # for vectorization, so we can "fold" the pairwise dimension
      # into the batch dimension, presenting it as
      # [batch*n_mol_1*n_mol_2, n_features]
      # to mlp2 and then recover it back
      B, N1, N_feat = a_broadcast.shape
      _B, N2, _N_feat = b_broadcast.shape
      a_batched = a_broadcast.reshape(B*N1*N2, N_feat)
      b_batched = b_broadcast.reshape(B*N1*N2, N_feat)
      # above, -1 would suffice instead of B*N1*N2, just being explicit
      batch_output = self.mlp2(a_batched, b_batched)

      # this should be exactly the same as `pairwise_classified`
      alternative_classified = batch_output.reshape(B, N1, N2, -1)

      return a_classified, b_classified, pairwise_classified


Instruction: RuntimeError: CUDA error: initialization error when calling torch.distributed.init_process_group using torch multiprocessing

Input:
I created a pytest fixture using decorator to create multiple processes (using torch multiprocessing) for running model parallel distributed unit tests using pytorch distributed. I randomly encountered the below CUDA initialization error all of a sudden (when I was trying to fix some unit tests logic). Since then, all my unit tests have been failing and I traced the failure back to my pytest fixture which calls torch.distributed.init_process_group(…).
Error traceback:
Process Process-1:
Traceback (most recent call last):
  File "/usr/lib64/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/usr/lib64/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/fsx-dev/FSxLustre20201016T182138Z/prraman/home/workspace/ws_M5_meg/src/M5ModelParallelism/test_script/commons_debug.py", line 34, in dist_init
    torch.distributed.init_process_group(backend, rank=rank, world_size=world_size, init_method=init_method)
  File "/usr/local/lib64/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 480, in init_process_group
    barrier()
  File "/usr/local/lib64/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 2186, in barrier
    work = _default_pg.barrier()
RuntimeError: CUDA error: initialization error

Below is the pytest fixture I created:
import os
import time
import torch
import torch.distributed as dist
from torch.multiprocessing import Process, set_start_method
import pytest

# Worker timeout *after* the first worker has completed.
WORKER_TIMEOUT = 120

def distributed_test_debug(world_size=2, backend='nccl'):
    """A decorator for executing a function (e.g., a unit test) in a distributed manner.
        This decorator manages the spawning and joining of processes, initialization of
        torch.distributed, and catching of errors.

        Usage example:
        @distributed_test_debug(worker_size=[2,3])
        def my_test():
            rank = dist.get_rank()
            world_size = dist.get_world_size()
            assert(rank < world_size)

    Arguments:
        world_size (int or list): number of ranks to spawn. Can be a list to spawn
        multiple tests.
    """
    def dist_wrap(run_func):
        """Second-level decorator for dist_test. This actually wraps the function. """
        def dist_init(local_rank,
                      num_procs,
                      *func_args, **func_kwargs):
            """Initialize torch.distributed and execute the user function. """
            os.environ['MASTER_ADDR'] = '127.0.0.1'
            os.environ['MASTER_PORT'] = '29503'
            os.environ['LOCAL_RANK'] = str(local_rank)
            # NOTE: unit tests don't support multi-node so local_rank == global rank
            os.environ['RANK'] = str(local_rank)
            os.environ['WORLD_SIZE'] = str(num_procs)
            master_addr = os.environ['MASTER_ADDR']
            master_port = os.environ['MASTER_PORT']
            rank = local_rank

            # Initializes the default distributed process group, and this will also initialize the distributed package.
            init_method = "tcp://"
            init_method += master_addr + ":" + master_port
            print('inside dist_init, world_size: ', world_size)
            torch.distributed.init_process_group(backend, rank=rank, world_size=world_size, init_method=init_method)
            print("rank={} init complete".format(rank))

            #torch.distributed.destroy_process_group()
            # print("rank={} destroy complete".format(rank))

            if torch.distributed.get_rank() == 0:
                print('> testing initialize_model_parallel with size {} ...'.format(
                    2))

            if torch.cuda.is_available():
                torch.cuda.set_device(local_rank)

            run_func(*func_args, **func_kwargs)

        def dist_launcher(num_procs,
                          *func_args, **func_kwargs):
            """Launch processes and gracefully handle failures. """

            # Spawn all workers on subprocesses.
            #set_start_method('spawn')
            processes = []
            for local_rank in range(num_procs):
                p = Process(target=dist_init,
                            args=(local_rank,
                                  num_procs,
                                  *func_args),
                            kwargs=func_kwargs)
                p.start()
                processes.append(p)

            # Now loop and wait for a test to complete. The spin-wait here isn't a big
            # deal because the number of processes will be O(#GPUs) << O(#CPUs).
            any_done = False
            while not any_done:
                for p in processes:
                    if not p.is_alive():
                        any_done = True
                        break

            # Wait for all other processes to complete
            for p in processes:
                p.join(WORKER_TIMEOUT)

            failed = [(rank, p) for rank, p in enumerate(processes) if p.exitcode != 0]
            for rank, p in failed:
                # If it still hasn't terminated, kill it because it hung.
                if p.exitcode is None:
                    p.terminate()
                    pytest.fail(f'Worker {rank} hung.', pytrace=False)
                if p.exitcode < 0:
                    pytest.fail(f'Worker {rank} killed by signal {-p.exitcode}',
                                pytrace=False)
                if p.exitcode > 0:
                    pytest.fail(f'Worker {rank} exited with code {p.exitcode}',
                                pytrace=False)

        def run_func_decorator(*func_args, **func_kwargs):
            """Entry point for @distributed_test(). """

            if isinstance(world_size, int):
                dist_launcher(world_size, *func_args, **func_kwargs)
            elif isinstance(world_size, list):
                for procs in world_size:
                    dist_launcher(procs, *func_args, **func_kwargs)
                    time.sleep(0.5)
            else:
                raise TypeError(f'world_size must be an integer or a list of integers.')

        return run_func_decorator

    return dist_wrap

Below is how I run it:
@distributed_test_debug(world_size=2)
def test_dummy():
    assert 1 == 1

I have seen some issues raised in the past when torch multiprocessing and CUDA not working well together, not sure if this is related to that. Perhaps a different way I should be creating my multiple processes to avoid this problem? Any help is appreciated.
I am using pytorch version: 1.8.0a0+ae5c2fe

Output:
Not sure how much its useful now but we were getting this error when running yolox script command
https://github.com/Megvii-BaseDetection/YOLOX
python -m yolox.tools.train -n yolox-s -d 8 -b 64 --fp16 -o [--cache]
However when we removed the --cache parameter the error was resolved.
Another issue we encountered was script freezing which was resolved with help from this post.
https://github.com/NVIDIA/nccl/issues/631
Note this was performed with 4 GPU setup on single machine.


Instruction: How to fix RuntimeError: Bool type is not supported by dlpack

Input:
I have been using the following code to get the default Cora dataset provided by DGL, but the following error suddenly occurred today.
The code was runned in CoLab (python 3.7 and Pytorch backend). I believed this is a error from the DGL update (since it had worked all the time before). However, I just wonder if there is anything we can do on our end to fix this?
Thanks.

Output:
It seems that an error is from torch update to 1.10.0. Reinstalling torch to 1.9.1 works for me. You can reinstall torch in colab as follows:
!pip install dgl==0.6.1
!pip install torch==1.9.1
import dgl
cora = dgl.data.CoraGraphDataset()


Instruction: What are the in_features and out_features supposed to be?

Input:
torch.nn.Linear(in_features, out_features, bias=True, device=None, dtype=None)
I have a dataset of [914,19] shape. should my in_features be 914? And I want to predict 5 different values so should my output feature be 5?
class NeuralNetwork(nn.Module):
    def __init__(self):
        super(NeuralNetwork, self).__init__()
        self.linear1 = nn.Linear(914,512)
        self.linear2 = nn.Linear(512,512)
        self.linear3 = nn.Linear(512,512)
        self.linear4 = nn.Linear(512,5)

    def forward(self, x):
        x = F.relu(self.linear1(x))
        x = F.relu(self.linear2(x))
        x = F.relu(self.linear3(x))
        x = self.linear4(x)

        return x
NeuralNet = NeuralNetwork()
print(NeuralNet)

Output:
Your input data is shaped (914, 19), assuming 914 refers to your batch size here, then the in_features corresponds to 19. This can be read as a tensor containing 914 19-feature-long input vectors.
In this case, the in_features of linear1 would be set to 19.


Instruction: what does mask_fill in pytorch do here

Input:
I have tensor named "k1" which is in shape 3,1,1,9 and also have p1 tensor in shape of 3,7,9,9 and I wanna know what does the line below do?
p1 = p1 .masked_fill(k1== 0, float("-1e30"))

Output:
As the documentation page describes it:

Tensor.masked_fill(mask, value)
Fills elements of self tensor with value where mask is True. The shape of mask must be broadcastable with the shape of the underlying tensor.

In your case it will place in p1 the value of float("-1e30") at the positions where k1 is equal to zero. Since k1 has singleton dimensions its shape will be broadcasted to the shape of p1.


Instruction: The same output value whatever is the input value for a Pytorch LSTM regression model

Input:
My dataset looks like the following:

on the left, my inputs, and on the right the outputs.
The inputs are tokenized and converted to a list of indices, for instance, the molecule input:
'CC1(C)Oc2ccc(cc2C@HN3CCCC3=O)C#N'
is converted to:
[28, 28, 53, 69, 28, 70, 40, 2, 54, 2, 2, 2, 69, 2, 2, 54, 67, 28, 73, 33, 68, 69, 67, 28, 73, 73, 33, 68, 53, 40, 70, 39, 55, 28, 28, 28, 28, 55, 62, 40, 70, 28, 63, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
I use the following list of chars as my map from strings to indices
cs = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',    'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z',
'0','1','2','3','4','5','6','7','8','9',
'=','#',':','+','-','[',']','(',')','/','\'
, '@','.','%']
Thus, for every char in the input string, there is an index, and if the length of the input string is less than the max length of all inputs which is 100, I complement with zeros. (like in the above-shown example)
My model looks like this:
class LSTM_regr(torch.nn.Module) :
    def __init__(self, vocab_size, embedding_dim, hidden_dim) :
        super().__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True)
        self.linear = nn.Linear(hidden_dim, 1)
        self.dropout = nn.Dropout(0.2)

    def forward(self, x, l):
        x = self.embeddings(x)
        x = self.dropout(x)
        lstm_out, (ht, ct) = self.lstm(x)
        return self.linear(ht[-1])
vocab_size = 76
model =  LSTM_regr(vocab_size, 20, 256)

My problem is, after training, every input I give to the model to test it, gives me the same output (i.e., 3.3318). Why is that?
My training loop:
def train_model_regr(model, epochs=10, lr=0.001):
    parameters = filter(lambda p: p.requires_grad, model.parameters())
    optimizer = torch.optim.Adam(parameters, lr=lr)
    for i in range(epochs):
        model.train()
        sum_loss = 0.0
        total = 0
        for x, y, l in train_dl:
            x = x.long()
            y = y.float()
            y_pred = model(x, l)
            optimizer.zero_grad()
            loss = F.mse_loss(y_pred, y.unsqueeze(-1))
            loss.backward()
            optimizer.step()
            sum_loss += loss.item()*y.shape[0]
            total += y.shape[0]

EDIT:
I figured it out, I reduced the learning rate from 0.01 to 0.0005 and reduced the batch size from 100 to 10 and it worked fine.
I think this makes sense, the model was training on large batch size, thus it was learning to output the mean always since that's what the loss function does.

Output:
Your LSTM_regr returns the last hidden state regardless of the true sequence length. That is, if your true sequence is of length 3, x is of length 100, and the output is the last hidden state after processing 97 padding elements.
You should compute the loss for the prediction that matches the true length of each sequence.


Instruction: RuntimeError: CUDA error: no kernel image is available for execution on the device after model.cuda()

Input:
I am working on this model:
class Model(torch.nn.Module):
    def __init__(self, sizes, config):
        super(Model, self).__init__()

        self.lstm = []
        for i in range(len(sizes) - 2):
            self.lstm.append(LSTM(sizes[i], sizes[i+1], num_layers=8))
        self.lstm.append(torch.nn.Linear(sizes[-2], sizes[-1]).cuda())
        self.lstm = torch.nn.ModuleList(self.lstm)

        self.config_mel = config.mel_features

    def forward(self, x):
        # convert to log-domain
        x = x.clip(min=1e-6).log10()

        for layer in self.lstm[:-1]:
            x, _ = layer(x)
            x = torch.relu(x)

        #x = torch_unpack_seq(x)[0]

        x = self.lstm[-1](x)
        mask = torch.sigmoid(x)

        return mask

and then:
model = Model(model_width, config)
model.cuda()

But I am getting this error:
File "main.py", line 29, in <module>
    Model.train(args)
  File ".../src/model.py", line 57, in train
    model.cuda()
  File ".../.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 637, in cuda
    return self._apply(lambda t: t.cuda(device))
  File ".../.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 530, in _apply
    module._apply(fn)
  File "/.../.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 530, in _apply
    module._apply(fn)
  File ".../.local/lib/python3.8/site-packages/torch/nn/modules/rnn.py", line 189, in _apply
    self.flatten_parameters()
  File ".../.local/lib/python3.8/site-packages/torch/nn/modules/rnn.py", line 175, in flatten_parameters
    torch._cudnn_rnn_flatten_weight(
RuntimeError: CUDA error: no kernel image is available for execution on the device
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.

I have no idea why it is happening. I am trying to push model and the inputs in cuda, and I understand if the error was due to some models in CPU and some in GPU. But that is not the case here. I found some pip install solution here: Pytorch CUDA error: no kernel image is available for execution on the device on RTX 3090 with cuda 11.1
but I cannot use it as I am trying to do the work in a remote repo where I don't have access to pip install.
Is there a way I can solve this?

Output:
I checked the latest torch and torchvision version with cuda from the given link. Stable versions list: https://download.pytorch.org/whl/cu113/torch_stable.html
Below versions solved the error,
pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 -f https://download.pytorch.org/whl/torch_stable.html
Reference: #49161


Instruction: About np.einsum

Input:
I don't understand how the following code realizes the transformation of dimensions? The shape of C is [2, 3, 3, 4]. How to realize the following matrix operation without einsum function?
import numpy as np
a = np.random.randint(0, 10, (2,3,4))
b = np.random.randint(0, 10, (3, 6, 4))
c = np.einsum('bld,hid-> blhd', a,b)

Output:
You can find more details in about einstein notation wikipedia
This means that you have indices b,l,h,i,d
this will iterate the indices to cover all the inputs and build the input
I will use capital letters for the arrays here to distinguish from the indices.
C[b,l,h,d] += A[b,l,d] * B[h,i,d]

The shape of the output can be determined as follows.
You take the index of each output axis and look for the same index in the input. For instance the first axis of C is indexed with b that is also used to index the first axis of A, thus assert C.shape[0] == A.shape[0]. Repeating for the other axes we have assert C.shape[1] == A.shape[1], assert C.shape[2] == B.shape[0], and assert C.shape[3] == A.shape[2], also assert C.shape[3] == B.shape[2].
Notice that the index i does not affect where the term will be added, each element of the output can be written as
C[b,l,h,d] = sum(A[b,l,d] * B[h,i,d] for i in range(B.shape[1]))

Notice also that i is not used to index A. So this could be also written as
C[b,l,h,d] = A[b,l,d] * B[h,:,d].sum();

Or if you want to use vectorized operation
first expanding then reducing
C = A[:,:,None,:] * B[None,None,:,:,:].sum(-2)

expanding reducing then expandin, possible because A does not use i
C = A[:,:,None,:] * B.sum(-2)[None,None,:,:]


Instruction: Why does torch.utils.save_image overwrite saved images in my folder?

Input:
I am trying an adversarial attack on 10 images and I need to save all the perturbed images in a folder. So, I used torch.utils.save_image in pytorch which works pretty fine. I expect all the images to be saved in the folder but instead, they are being overwritten and the last image seen is the only image saved. I have the following attack() function that takes a single image to perturb
def attack(img, label, net, target=None, pixels=1, maxiter=75, popsize=400, verbose=False):
    # img: 1*3*W*H tensor
    # label: a number

    targeted_attack = target is not None
    target_calss = target if targeted_attack else label

    bounds = [(0,32), (0,32), (0,255), (0,255), (0,255)] * pixels

    popmul = max(1, popsize//len(bounds))

    predict_fn = lambda xs: predict_classes(
        xs, img, target_calss, net, target is None)
    callback_fn = lambda x, convergence: attack_success(
        x, img, target_calss, net, targeted_attack, verbose)

    inits = np.zeros([popmul*len(bounds), len(bounds)])

    count = 1
    for init in inits:
        for i in range(pixels):
            init[i*5+0] = np.random.random()*32
            init[i*5+1] = np.random.random()*32
            init[i*5+2] = np.random.normal(128,127)
            init[i*5+3] = np.random.normal(128,127)
            init[i*5+4] = np.random.normal(128,127)

    attack_result = differential_evolution(predict_fn, bounds, maxiter=maxiter, popsize=popmul,
        recombination=1, atol=-1, callback=callback_fn, polish=False, init=inits)

    attack_image = perturb_image(attack_result.x, img)
    # attack_var = Variable(attack_image, volatile=True).cuda()
    with torch.no_grad():
        attack_var = attack_image.to(device)
        predicted_probs = F.softmax(net(attack_var), dim=1).data.cpu().numpy()[0]

        predicted_class = np.argmax(predicted_probs)

    vutils.save_image(vutils.make_grid(attack_image, normalize=True, scale_each=True), 'result_img/adversarial' + str(count) + '.png')
    vutils.save_image(vutils.make_grid(img, normalize=True, scale_each=True), 'result_img/original' + str(count) + '.png')
    count = count + 1

    if (not targeted_attack and predicted_class != label) or (targeted_attack and predicted_class == target_calss):
        return 1, attack_result.x.astype(int)
    return 0, [None]

Below is the attack_all() function that perturbs batches of images (entire test set) which is 10 images in my case.
def attack_all(net, loader, pixels=1, targeted=False, maxiter=75, popsize=400, verbose=False):

    correct = 0
    success = 0

    for batch_idx, (input, target) in enumerate(loader):

        # img_var = Variable(input, volatile=True).cuda()
        with torch.no_grad():
            img_var = input.to(device)
            target = target

            prior_probs = F.softmax(net(img_var), dim=1)
            _, indices = torch.max(prior_probs, 1)

            if target[0] != indices.data.cpu()[0]:
                continue

            correct += 1
            target = target.numpy()

            targets = [None] if not targeted else range(10)

        for target_calss in targets:
            if (targeted):
                if (target_calss == target[0]):
                    continue

            flag, x = attack(input, target[0], net, target_calss, pixels=pixels, maxiter=maxiter, popsize=popsize, verbose=verbose)

            success += flag
            if (targeted):
                success_rate = float(success)/(9*correct)
            else:
                success_rate = float(success)/correct

            if flag == 1:
                print("success rate: %.4f (%d/%d) [(x,y) = (%d,%d) and (R,G,B)=(%d,%d,%d)]"%(
                    success_rate, success, correct, x[0],x[1],x[2],x[3],x[4]))

        if correct == args.samples:
            break
    return success_rate

Below is the main() class where I am attacking the 10 images with attack_all(). I expect all the 10 images (both original and perturbed) to be saved but only the last seen image is saved.
def main():

    print ("==> Loading data and model...")
    transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
        ])
    # test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=tranfrom_test)
    test_set = Cifar10Dataset(csv_file='mydata/cifar10.csv', root_dir = 'mydata/cifar_selected_10', transform = transform_test)
    testloader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=True, num_workers=2)

    class_names = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
    assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
    checkpoint = torch.load('./checkpoint/%s.t7'%args.model)
    net = checkpoint['net']
    net.cuda()
    cudnn.benchmark = True

    print ("==> Starting attack...")

    results = attack_all(net, testloader, pixels=args.pixels, targeted=args.targeted, maxiter=args.maxiter, popsize=args.popsize, verbose=args.verbose)
    print ("Final success rate: %.4f"%results)

Output:
So I figured out how to solve it myself.
I noticed that variable count in attack() will not increase no matter how. Instead, I set count = 1 outside attack() and did global count inside same attack(). This way, value of count can change and will not remain thesame everytime attack_all() calls the function attack().


Instruction: How do I load a yolov5l model with custom weights into torch in python?

Input:
I have trained a yolov5l model for object detection and classification. I want to use the exported weights to identify images in a program I am creating. I am having trouble finding much of anything on how to use .pt weights in a python program.
I believe I use the "torch.load" method from the pytorch library, but when I try: torch.load(path_to_weights)
I get a ModuleNotFoundError for not having a module named 'models'.
Any help is much appreciated. Thank you very much.

Output:
You should use torch.load_state_dict() method to load your trained parameters to your model in addition to torch.load().
There are some issues with your torch.load() method. You should provide your path parameter as a either string or os.PathLike object. (These are written in the docs).
I am going to provide a simple code block to show you the way.
#Initializing model

model = Model() # Assuming your model's name is Model

model.load_state_dict(torch.load(path_to_weights))

But don't forget that your path_to_weights parameter must be either string or os.PathLike object.


Instruction: RuntimeError: PytorchStreamReader failed locating file data.pkl: file not found

Input:
I have been trying to train some data using a model that utilizes src+img. When running the training script, I'm running into an error namely: RuntimeError: PytorchStreamReader failed locating file data.pkl: file not found
the .pkl file in this should have been the pickled .pt file.
First I figured that maybe I did not specify the preprocessed training.pt file correctly, but it is actually correctly specified.
data.pkl hasn't been dumped anywhere (or I failed to find it). I am guessing it has something do with pickle accordimg to docs:
"This save/load process uses the most intuitive syntax and involves the least amount of code. Saving a model in this way will save the entire module using Python’s pickle module. The disadvantage of this approach is that the serialized data is bound to the specific classes and the exact directory structure used when the model is saved. The reason for this is because pickle does not save the model class itself. Rather, it saves a path to the file containing the class, which is used during load time. Because of this, your code can break in various ways when used in other projects or after refactors."
I've tried a multitude of things, like changing data.pkl in the script in order to see if there was a generated corrupted file, but this seems to not be the case.
I would appreciate it if anyone is willing to help:
full error:
  File "train_mm.py", line 448, in <module>
    main()
  File "train_mm.py", line 423, in main
    first_dataset = next(lazily_load_dataset("train"))
  File "train_mm.py", line 314, in lazily_load_dataset
    yield lazy_dataset_loader(pt, corpus_type)
  File "train_mm.py", line 305, in lazy_dataset_loader
    dataset = torch.load(pt_file)
  File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 607, in load
    return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
  File "/usr/local/lib/python3.7/dist-packages/torch/serialization.py", line 878, in _load
    data_file = io.BytesIO(zip_file.get_record(pickle_file))
RuntimeError: PytorchStreamReader failed locating file data.pkl: file not found

solved: I re-ran the preprocessing script and it generated new .pt files and the error was resolved

Output: This issue has been resolved. the .pt file was heavily corrupted. After deleting the corrupt .pt file and re-running the preprocess script and consequently training script, I did not get the error anymore.


Instruction: What is the difference between cuda.amp and model.half()?

Input:
According to https://pytorch.org/blog/accelerating-training-on-nvidia-gpus-with-pytorch-automatic-mixed-precision/
We can use:
   with torch.cuda.amp.autocast():
      loss = model(data)

In order to casts operations to mixed precision.
Another thing is that we can use  model.half() to convert all the model weights to half precision.

What is the difference between these 2 commands ?
If I want to take advantage of FP16 (in order to create larger models and shorter training time), what do I need ?
Do I need to use  model.half() or using torch.cuda.amp (according the link above) ?

Output:
If you convert the entire model to fp16, there is a chance that some of the activations functions and batchnorm layers will cause the fp16 weights to underflow, i.e., become zero. So it is always recommended to use autocast which internally converts the weights to fp32 in problematic layers.
model.half() in the end will save weight in fp16 where as autocast weights will be still in fp32. Training in fp16 will be faster than autocast but higher chance for instability if you are not careful.
While using autocast you also need to scale up the gradient during the back propagation
If fp16 requirement is on the inference side, I recommend using autocast and then converting to fp16 using ONNX and tensorrt.


Instruction: How to save png images with OpenCV

Input:
I am trying to use OpenCV to convert the results of the model I trained into a png image. My output has 4 channels, and I am not sure how to convert these 4 channels to png.
# Load the model
model = CNNSEG()
model.load_state_dict(torch.load(PATH))
model.eval()

for iteration, sample in enumerate(test_data_loader):
    img = sample
    print(img.shape)

    plt.imshow(img[0,...].squeeze(), cmap='gray') #visualise all images in test set
    plt.pause(1)

    # output the results
    img_in = img.unsqueeze(1)
    output = model(img_in) # shape: [2, 4, 96, 96]

As here shows, the shape of output is [2, 4, 96, 96] which are batch size, channels, height and width. So how could I do to convert it to png image?

Output:
You would want to split your image into two essentially and then save them individually.
import numpy as np
import cv2
img = np.ones((2,4,96,96),dtype=np.uint8) #creating a random image

img1 = img[0,:,:,:] #extracting the two separate images
img2 = img[1,:,:,:]

img1_reshaped = img1.transpose() #reshaping them to the desired form of (w,h,c)
img2_reshaped = img2.transpose()

cv2.imwrite("img1.png",img1_reshaped) #save the images as .png
cv2.imwrite("img2.png",img2_reshaped)


Instruction: need help in modifying code to access YOLOv5 for CPU

Input:
I am trying to implement a object detection program with Pytorch , OpenCV and YOLOv5 that detect the objects and the type of object from a YouTube video . However, while running, the output console shows that the YOLO version the program is trying to run is for CUDA . I wish to use the YOLO for CPU to implement the project as my computer is not set up for CUDA. I seek assistance on how may I go about modifying the program to utilize the YOLO for CPU ? Thank You Very Much !
Link to GitHub Gist
https://gist.github.com/neitherjames/c3b91033eca3794f8170ee51ee5357d4

Output: From your __init__ it shows that you give the system the option to choose CUDA if it is available. You can force it to run on cpu by stating self.device = 'cpu' on line 23. Then when calling self.model.to(self.device) on line 48, the model is sent to cpu.


Instruction: pytorch RNN loss does not decrease and validate accuracy remains unchanged

Input:
I'm training a model using Pytorch GRU on a text-classification task (output dimension is 5). My network is implemented like the codes below.
class GRU(nn.Module):

    def __init__(self, model_param: ModelParam):
        super(GRU, self).__init__()

        self.embedding = nn.Embedding(model_param.vocab_size, model_param.embed_dim)

        # Build with pre-trained embedding vectors, if given.
        if model_param.vocab_embedding is not None:
            self.embedding.weight.data.copy_(model_param.vocab_embedding)
            self.embedding.weight.requires_grad = False

        self.rnn = nn.GRU(model_param.embed_dim,
                          model_param.hidden_dim,
                          num_layers=2,
                          bias=True,
                          batch_first=True,
                          dropout=0.5,
                          bidirectional=False)

        self.dropout = nn.Dropout(0.5)

        self.fc = nn.Sequential(
            nn.Linear(in_features=model_param.hidden_dim, out_features=128),
            nn.Linear(in_features=128, out_features=model_param.output_dim)
        )

    def forward(self, x, labels=None):
        '''
            :param x: torch.tensor, of shape [batch_size, max_seq_len].
            :param labels: torch.tensor, of shape [batch_size]. Not used in this model.
            :return outputs: torch.tensor, of shape [batch_size, output_dim].
        '''

        # [batch_size, max_seq_len, embed_dim].
        features = self.dropout(self.embedding(x))

        # [batch_size, max_seq_len, hidden_dim].
        outputs, _ = self.rnn(features)

        # [batch_size, hidden_dim].
        outputs = outputs[:, -1, :]

        return self.fc(self.dropout(outputs))

I'm using nn.CrossEntropyLoss() for loss function, and optim.SGD for optimizer. The definition of loss function and optimizer is given like this.
# Loss function and optimizer.
loss_func = nn.CrossEntropyLoss()
optimizer = SGD(model.parameters(), lr=learning_rate, weight_decay=0.9)

And my training procedure is roughly shown as below.
            for batch in train_iter:

                optimizer.zero_grad()

                # The prediction of model, and its corresponding loss.
                prediction = model(batch.text.type(torch.LongTensor).to(device), batch.label.to(device))
                loss = loss_func(prediction, batch.label.to(device))

                loss.backward()
                optimizer.step()

                # Record total loss.
                epoch_losses.append(loss.item() / batch_size)

When I'm training this model, validate accuracy and losses are reported like this.
Epoch 1/300 valid acc: [0.839] (16668 in 19873), time spent 631.497 sec. Validate loss 1.506138. Best validate epoch is 1.
Epoch 2/300 valid acc: [0.839] (16668 in 19873), time spent 627.631 sec. Validate loss 1.577007. Best validate epoch is 2.
Epoch 3/300 valid acc: [0.839] (16668 in 19873), time spent 631.427 sec. Validate loss 1.580756. Best validate epoch is 3.
Epoch 4/300 valid acc: [0.839] (16668 in 19873), time spent 605.352 sec. Validate loss 1.581306. Best validate epoch is 4.
Epoch 5/300 valid acc: [0.839] (16668 in 19873), time spent 388.487 sec. Validate loss 1.581431. Best validate epoch is 5.
Epoch 6/300 valid acc: [0.839] (16668 in 19873), time spent 360.344 sec. Validate loss 1.581464. Best validate epoch is 6.
Epoch 7/300 valid acc: [0.839] (16668 in 19873), time spent 624.345 sec. Validate loss 1.581473. Best validate epoch is 7.
Epoch 8/300 valid acc: [0.839] (16668 in 19873), time spent 622.059 sec. Validate loss 1.581477. Best validate epoch is 8.
Epoch 9/300 valid acc: [0.839] (16668 in 19873), time spent 651.425 sec. Validate loss 1.581478. Best validate epoch is 9.
Epoch 10/300 valid acc: [0.839] (16668 in 19873), time spent 697.475 sec. Validate loss 1.581478. Best validate epoch is 10.
...

It shows that validate loss does not decrease after epoch 9, and validate accuracy keeps unchanged since the first epoch (Note that in my dataset, one of the labels accounted for 83%, it could be inferred from this that my model tends to prediction all sequences to the same label, but this also happens when I'm training on another dataset that is relatively less unbalanced). Are there anybody that has met this situation B4? I'm wondering if I have made a mistake in designing model or training procedure. Thanks for your help XD.
Updated on Nov.19th, I have added a figure which shows how loss behaves while training. It can be known from this figure that both training loss and validating loss turned to be constant after 5th epoch.
training and validating loss in 20 epochs

Output:
Now I found that loss does not drop down mainly because weight decay set in optimizer is to high.
optimizer = SGD(model.parameters(), lr=learning_rate, weight_decay=0.9)

So I fixed this and changed weight decay to be 5e-5.
optimizer = SGD(model.parameters(), lr=learning_rate, weight_decay=5e-5)

This time the loss of my network begins to decrease. However, there is no improvement in accuracy.
Epoch 1/100 valid acc: [0.839] (16668 in 19873), time spent 398.154 sec. Validate loss 0.713456. Best validate epoch is 1.
Epoch 2/100 valid acc: [0.839] (16668 in 19873), time spent 572.057 sec. Validate loss 0.631721. Best validate epoch is 2.
Epoch 3/100 valid acc: [0.839] (16668 in 19873), time spent 580.867 sec. Validate loss 0.613186. Best validate epoch is 3.
Epoch 4/100 valid acc: [0.839] (16668 in 19873), time spent 561.953 sec. Validate loss 0.601883. Best validate epoch is 4.
Epoch 5/100 valid acc: [0.839] (16668 in 19873), time spent 564.913 sec. Validate loss 0.596573. Best validate epoch is 5.
Epoch 6/100 valid acc: [0.839] (16668 in 19873), time spent 574.525 sec. Validate loss 0.592848. Best validate epoch is 6.
Epoch 7/100 valid acc: [0.839] (16668 in 19873), time spent 580.885 sec. Validate loss 0.591074. Best validate epoch is 7.
Epoch 8/100 valid acc: [0.839] (16668 in 19873), time spent 455.228 sec. Validate loss 0.589787. Best validate epoch is 8.
Epoch 9/100 valid acc: [0.839] (16668 in 19873), time spent 582.756 sec. Validate loss 0.588691. Best validate epoch is 9.
Epoch 10/100 valid acc: [0.839] (16668 in 19873), time spent 583.997 sec. Validate loss 0.588260. Best validate epoch is 10.
Epoch 11/100 valid acc: [0.839] (16668 in 19873), time spent 599.630 sec. Validate loss 0.588224. Best validate epoch is 11.
Epoch 12/100 valid acc: [0.839] (16668 in 19873), time spent 597.713 sec. Validate loss 0.586977. Best validate epoch is 12.
Epoch 13/100 valid acc: [0.839] (16668 in 19873), time spent 605.038 sec. Validate loss 0.587937. Best validate epoch is 13.
Epoch 14/100 valid acc: [0.839] (16668 in 19873), time spent 598.712 sec. Validate loss 0.587059. Best validate epoch is 14.
Epoch 15/100 valid acc: [0.839] (16668 in 19873), time spent 409.344 sec. Validate loss 0.587293. Best validate epoch is 15.
...

How training loss behaves is shown in this figure.

I'm wondering if learning rate of 1e-3 and weight decay of 5e-5 are reasonable settings. My designated size of batch is 32.


Instruction: PyTorch and torch_scatter were compiled with different CUDA versions on Google Colab despite attempting to specify same version

Input:
I'm installing pytorch geometric on Google colab. I've done this lots of times before and had no issues but it has suddenly stopped working. I've not changed my code since it worked. Here is how I install it:
!pip install torch==1.8.1 torchvision torchtext
import torch; print(torch.__version__); print(torch.version.cuda)

!pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html
!pip install torch-sparse -f https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html
!pip install torch-cluster -f https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html
!pip install torch-spline-conv -f https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html
!pip install torch-geometric

The pytorch version should be 1.8.1+cu102, confirmed by the print statement above. I specify the version when installing with !pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html. However, when I import torch_geometric I get the error:

Detected that PyTorch and torch_scatter were compiled with different CUDA versions. PyTorch has CUDA version 10.2 and torch_scatter has CUDA version 11.1. Please reinstall the torch_scatter that matches your PyTorch install.

Why is torch_scatter not compiling with CUDA version 10.2? Is there a way to force it to compile with this version?

Output:
you could try to specify the latest wheel version provided by the link you use: https://pytorch-geometric.com/whl/torch-1.8.1+cu102.html (for November 22nd 2021 it is 2.0.8):
pip install torch-scatter==2.0.8 -f https://data.pyg.org/whl/torch-1.8.1+cu102.html

It looks like the latest torch-scatter version in Google Colab is 2.0.9 which is newer than 2.0.8. Therefore, when you run your command it does not do anything thinking that the latest version is already installed.


Instruction: How to use tensor cores in pytorch and tensorflow?

Input: I am using a Nvidia RTX GPU with tensor cores, I want to make sure pytorch/tensorflow is utilizing its tensor cores. I noticed in few articles that the tensor cores are used to process float16 and by default pytorch/tensorflow uses float32. They have introduced some lib that does "mixed precision and distributed training". It is a somewhat old answer. I want to know if pytorch or tensorflow GPU now supports tensor core processing out of the box.

Output:
Mixed Precision is available in both libraries.
For pytorch it is torch.cuda.amp, AUTOMATIC MIXED PRECISION PACKAGE.
https://pytorch.org/docs/stable/amp.html
https://pytorch.org/docs/stable/notes/amp_examples.html.
Tensorflow has it here, https://www.tensorflow.org/guide/mixed_precision.


Instruction: tensorflow 2.4.1 requires six~=1.15.0, but you have six 1.16.0 which is incompatible

Input:
Iam getting this error:-
    [31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
104 1m 15s  tensorflow 2.4.1 requires six~=1.15.0, but you have six 1.16.0 which is incompatible.[

when i try to import packages torch and datasets for huggingface summarization in my client's environment.
pandas
numpy
torch
datasets==1.14.0
transformers==4.11.3
rouge-score==0.0.4
nltk==3.6.5
pyarrow==6.0.0

beautifulsoup4==4.10.0
numpy
pandas
lxml
requests==2.23.0
wikipedia==1.4.0

These are the packages that I am giving as requirements. Could it be that one of these is installing six=1.16 and uninstalling six=1.15?
Here is the further detail of the log
Downloading cnvrg-0.7.51-py3-none-any.whl (78 kB)
24  5m 22s  [?25l
25  5m 22s  [?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (1.15.0)
26  5m 22s  Collecting six
27  5m 22s  Downloading six-1.16.0-py2.py3-none-any.whl (11 kB)
[?25l

Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2.0dev,>=1.11.0->google-cloud-storage->cnvrg) (0.4.8)
83  5m 22s  Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from pyaml->cnvrg) (5.4.1)
84  5m 22s  Installing collected packages: six, urllib3, requests, azure-core, azure-storage-blob, cnvrg
85  5m 22s  Attempting uninstall: six
86  5m 22s  Found existing installation: six 1.15.0
87  5m 22s  Uninstalling six-1.15.0:
88  5m 22s  Successfully uninstalled six-1.15.0

Eventually, this error is not allowing me to import both torch and datasets.
Can anyone help in resolving this

Output:
after installing all dependencies, install six 1.15.0
pip install -r requirements.txt

then run
pip install six~=1.15.0

or
pip install six==1.15.0


Instruction: Best way to import CSV data in a python tensor for machine learning?

Input: I want to import a csv data file in python in order to create a neural network down the road (with pytorch). The file has 4 columns and around 1000 rows with first row as titles. Which is the best way to do this?

Output:
Just use pandas.
In particular what you need is the read_csv function.
import pandas as pd

...

dataframe = pd.read_csv("/location/file.csv")

Check out the pandas references for more details.


Instruction: Pandas: Unexpected behavior for apply function with torch.tensor()

Input:
I confused of the behavior of the panda.apply() function. I want to convert a column containing a list of int to a troch.tensor. Here is some sample code showing the behavior:
df_test = pd.DataFrame([3,3,3], columns=['value'])
df_test.value = df_test.value.apply(lambda x: [y for y in range(x)])
print(df_test)
# Output:
#        value
# 0  [0, 1, 2]
# 1  [0, 1, 2]
# 2  [0, 1, 2]

print(df_test.value.apply(lambda x: torch.tensor(x)))
# Output:
#                                value
# 0  [tensor(0), tensor(1), tensor(2)]
# 1  [tensor(0), tensor(1), tensor(2)]
# 2  [tensor(0), tensor(1), tensor(2)]

print(df_test.value.apply(lambda x: x + [12]))
# Output:
# 0    [0, 1, 2, 12]
# 1    [0, 1, 2, 12]
# 2    [0, 1, 2, 12]

print(torch.tensor([1,2,3]))
# Output:
# tensor([1, 2, 3])

I would have expected, one tensor with three elements per row element, but instead the apply creates a list of tensors containing one element. For testing, I added an example that adds an element to the list, to ensure, that x is the list itself. As you can see it behaves as expected. Can anyone explain the behavior?
Is there a workaround? I don't want to use torch.tensor(df.values), since I need to apply the tensor transformation to multiple columns and want to keep them in the dataframe. Thanks!

Output:
The reason is that apply function converts implicitly a tensor to list because the type of df_test.value[0] is a list. When you convert a tensor to a list, here is a result:
print(df_test.value[0])  # list
x = torch.tensor([1,2,3])
print(list(x))    # convert a tensor to a list

[tensor(1), tensor(2), tensor(3)]

You expected tensor([1, 2, 3]) replacing each list in df_test["value"]. But do not forget the column type will be tensor, which is not valid type in pandas.
To solve this problem is to convert a dataframe to NumPy array and then to a tensor. Then you can do all your transformations and then convert it again to NumPy to pandas.
If you try this code:
df_test["new"]= torch.tensor([1,2,3])
type(df_test.new.dtype) # it is not tensor but NumPy which is implicit conversion

numpy.dtype[int64]


Instruction: How to convert from tensor to float

Input:
I am having a list of tensors I want to convert to floating points how can I do it. I have tried using .item but it is not working. I am getting ValueError: only one element tensors can be converted to Python scalars.
tensor([[12.1834,  4.9616,  7.7913],
        [ 8.9394,  8.5784,  9.3691],
        [ 9.4475,  8.9766,  9.8418],
        [11.8210,  6.0852,  8.2168],
        [ 8.2885,  6.2607,  9.8877]], grad_fn=<CloneBackward0>)

Output:
You just need to cast Tensor constant to numpy object,then can access by index.
result.numpy()[0]


Instruction: Pytorch - IndexError: index out of range in self

Input:
I am working on building a LSTM based seq2seq sentence - slots solution.
For instance:
Input sentence: My name is James Bond
Output Slot: O O O B-name I-name
I'm unable to figure out the reason for the below error:
IndexError: index out of range in self
> <ipython-input-37-19283c592e18>(12)<module>()
     10     set_trace()
     11     inputs = torch.tensor(training_data[0][0])
---> 12     tag_scores = model(inputs)
     13     print(tag_scores)

When I try to execute the following code -
class LSTMTagger(nn.Module):

    def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size):
        super(LSTMTagger, self).__init__()
        self.hidden_dim = hidden_dim
        self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim)
        self.hidden2tag = nn.Linear(hidden_dim, tagset_size)

    def forward(self, sentence):
        embeds = self.word_embeddings(sentence)
        lstm_out, _ = self.lstm(embeds.view(len(sentence), 1, -1))
        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))
        tag_scores = F.log_softmax(tag_space, dim=1)
        return tag_scores

model = LSTMTagger( EMBEDDING_DIM, HIDDEN_DIM, len(vocab2sent), len(vocab2slot))
loss_function = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.1)

with torch.no_grad():
    inputs = torch.tensor(training_data[0][0])
    tag_scores = model(inputs)
    print(tag_scores)

for epoch in range(300):
    for sentence, tags in training_data:
        model.zero_grad()
        sentence_in = torch.tensor(sentence, dtype=torch.long)
        targets = torch.tensor(tags, dtype=torch.long)
        tag_scores = model(sentence_in)
        loss = loss_function(sentence_in, targets)
        loss.backward()
        optimizer.step()
with torch.no_grad():
    inputs = prepare_sequence(training_data[0][0], vocab2sent)
    tag_scores = model(inputs)
    print(tag_scores)

My variable values:
vocab2sent - dict with input sentences vocabulary ( word : unique number)
vocab2slot - dict with output vocabulary (slot : unique number)
inputs - tensor([ 229, 1056,  701,  330, 1093,   37,  166,  517, 1150, 1150, 1150, 1150,
        1150, 1150, 1150, 1150, 1150, 1150, 1150, 1150, 1150])
Model value during runtime -
LSTMTagger(
  (word_embeddings): Embedding(1148, 560)
  (lstm): LSTM(560, 560)
  (hidden2tag): Linear(in_features=560, out_features=28, bias=True)
)

Output: The vocabulary size for the Embedding layer is 1148: Embedding(1148, 560) but in the inputs you have index 1150. Maybe it is the source of your problem?


Instruction: How to build neural network in this structure？with different nodes connects to different number of nodes in next layer

Input:
I only know how to use the built-in network like RNN of LSTM in PyTorch. But they tend to deal with every node in the previous layer that will give information to all nodes in the next layer.
I want to do something different but don't know how to code it myself. Like in this figure: the node a maps to all [d, e, f]  three nodes in layer 2, while node b maps to [e,f] and node c only maps to [f]. As a result,  node d will only contain information from a,  while e will contain information from  [a, b]. And f will contain information from all nodes in the previous layer.
Does anyone know how to code this structure? PLS give me some insight I'll be very grateful :D
Structure

Output:
When you have a layer that looks like Fully-Connected layer but with custom connectivity, use a mask with proper structure.
Let's say x = [a, b, c] is your 3-dim input and W denotes the connectivity matrix.
>> x
tensor([[0.1825],
        [0.9598],
        [0.2871]])
>> W
tensor([[0.7459, 0.4669, 0.9687],
        [0.9016, 0.4690, 0.0471],
        [0.5926, 0.9700, 0.5222]])

then W[i][j] points to the connecting weight between jth input and ith output neuron. To build the structure similar to your toy example, we would make a mask like this
>> mask
tensor([[1., 0., 0.],
        [1., 1., 0.],
        [1., 1., 1.]])

Then you can simply mask the W
>> (mask * W) @ x
tensor([[0.1361],
        [0.6147],
        [1.1892]])

Note: @ is matrix multiplication and * is pointwise multiplication.


Instruction: loss.backward() no grad in pytorch NN

Input:
The code gives an error in loss.backward()
Error is:
untimeError: element 0 of tensors does not require grad and does not have a grad_fn
for epoch in range(N_EPOCHS):
    model.train()
    for i,(im1, im2, labels) in enumerate(train_dl):
        i1 = torch.flatten(im1,1)
        i2 = torch.flatten(im2,1)
        inp = torch.cat([i1,i2],1)

        b_x = Variable(inp) # batch x
        b_y = Variable(labels) # batch y
        y_ = model(b_x).squeeze()
        y_ = (y_>0.5).float()

        print(y_)
        print(l)
        loss = criterion(y_,b_y)
        print(loss.item())
        loss.backward()
        optimizer.step()

Output:
With additional info given by OP in the comment, the correct approach here is just removing the line
y_ = (y_>0.5).float()


Instruction: Select and MSELoss for two torch tensors

Input:
I have two torch tensors:
predictions = tensor([[33, 34,  7,  5,  5, 23, 22,  1,  3,  5, 23,  1],
        [14,  1, 22,  7,  5, 11,  7, 33,  3, 12, 25, 22],
        [33,  1, 14, 12, 23, 22, 12,  2,  3, 12, 23, 14],
        [23, 34, 34,  3,  5, 25, 12, 11,  2, 23, 23, 13]])

labels = tensor([[-100, -100, -100, -100, -100,   11, -100, -100, -100, -100, -100, -100],
        [-100, -100, -100, -100, -100, -100, -100, -100,   40, -100, -100, -100],
        [-100,   42, -100,   43, -100, -100, -100, -100, -100, -100, -100, -100],
        [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100,   32, -100]])

I'd like to calculate MSELoss(predictions, labels) only for values not equal to -100 - but don't know how to select corresponding values. For this example the result should be calculated for following values:
[23] and [11]
[3]  and [40]
[1,12] and [42, 43]
[23]  and [32]

I trying like this:
nn.MSELoss(preds.squeeze(), labels.squeeze())

but receive error.

Output:
In Short
# convert to float
predictions = predictions.to(torch.float)
labels = labels.to(torch.float)

# pick the right entries
reduced_predictions = predictions[labels != -100]
reduced_labels = labels[labels != -100]

# initialaize loss_fn
torch.nn.MSELoss()(reduced_predictions, reduced_labels)
# Note          ^     calling the nn.MSELoss() and passing tensors in additional ()

In Detail
First you need to filter the undesired entries, you can do so as follows:
import torch

predictions = torch.tensor([[33, 34,  7,  5,  5, 23, 22,  1,  3,  5, 23,  1],
        [14,  1, 22,  7,  5, 11,  7, 33,  3, 12, 25, 22],
        [33,  1, 14, 12, 23, 22, 12,  2,  3, 12, 23, 14],
        [23, 34, 34,  3,  5, 25, 12, 11,  2, 23, 23, 13]])

labels = torch.tensor([[-100, -100, -100, -100, -100,   11, -100, -100, -100, -100, -100, -100],
        [-100, -100, -100, -100, -100, -100, -100, -100,   40, -100, -100, -100],
        [-100,   42, -100,   43, -100, -100, -100, -100, -100, -100, -100, -100],
        [-100, -100, -100, -100, -100, -100, -100, -100, -100, -100,   32, -100]])

# find the entries different than -100
indices = labels != -100

# pick the corresponding values from predictions and labels
reduced_predictions = predictions[indices]
reduced_labels = labels[indices].to(torch.float)

The nn.MSELoss() receives floats so you must convert your tensors to a suitable dtype.
Then, you have to instantiate the loss function (create an instance of MSELoss by calling it's initiator):
loss_fn = torch.nn.MSELoss()

And only then call it using your tensors:
loss_fn(reduced_predictions, reduced_labels)

The result I received: tensor(847.2000)


Instruction: Create a Image stack in PyTorch

Input:
I have a Bx3xHxW image tensor in PyTorch and wish to create a Bx3FxHxW image stack of this image where F=64.
The image stack is formed by shifting the original image right. That is if the original image is to be shifted by 2 pixels to the right, the two left-most columns in new image will be 0 and the third-last column of the original image will become the last column of the new shifted image.
The image stack is formed by shifting right the original image f times where f varies from 0 to F-1.
How to achieve this in PyTorch in the most effecient manner using no or minimal number of for loops? A numpy code will also work as the two languages are quite compatible.

Output:
Torch (as well as numpy) provides torch.roll function, by padding with zeros first, rolling and then slicing the result you can achieve your right shift.
Here's a numpy version:
import numpy as np

X = np.random.rand(4,3,28,28)
Z = np.zeros((4,3,28,28))
XZ = np.concatenate([X,Z],axis=-1)

res = []
shift = 2
F = 28//shift
for i in range(0,F,shift):
    res.append(np.roll(XZ,i,-1)[:,:,:,:28])

res = np.concatenate(res,axis=1)

Here's a 1D example for better understanding:
x = np.arange(10)
z = np.zeros(10)
xz = np.concatenate([x,z],axis=-1)

for i in range(0,10,2):
    print(np.roll(xz,i,-1)[:10])

[0. 1. 2. 3. 4. 5. 6. 7. 8. 9.]
[0. 0. 0. 1. 2. 3. 4. 5. 6. 7.]
[0. 0. 0. 0. 0. 1. 2. 3. 4. 5.]
[0. 0. 0. 0. 0. 0. 0. 1. 2. 3.]
[0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]

I hope this is what you are looking for.


Instruction: Understanding torch.nn.LayerNorm in nlp

Input:
I'm trying to understanding how torch.nn.LayerNorm works in a nlp model. Asuming the input data is a batch of sequence of word embeddings:
batch_size, seq_size, dim = 2, 3, 4
embedding = torch.randn(batch_size, seq_size, dim)
print("x: ", embedding)

layer_norm = torch.nn.LayerNorm(dim)
print("y: ", layer_norm(embedding))

# outputs:
"""
x:  tensor([[[ 0.5909,  0.1326,  0.8100,  0.7631],
         [ 0.5831, -1.7923, -0.1453, -0.6882],
         [ 1.1280,  1.6121, -1.2383,  0.2150]],

        [[-0.2128, -0.5246, -0.0511,  0.2798],
         [ 0.8254,  1.2262, -0.0252, -1.9972],
         [-0.6092, -0.4709, -0.8038, -1.2711]]])
y:  tensor([[[ 0.0626, -1.6495,  0.8810,  0.7060],
         [ 1.2621, -1.4789,  0.4216, -0.2048],
         [ 0.6437,  1.0897, -1.5360, -0.1973]],

        [[-0.2950, -1.3698,  0.2621,  1.4027],
         [ 0.6585,  0.9811, -0.0262, -1.6134],
         [ 0.5934,  1.0505, -0.0497, -1.5942]]],
       grad_fn=<NativeLayerNormBackward0>)
"""

From the document's description, my understanding is that the mean and std are computed by all embedding values per sample. So I try to compute y[0, 0, :] manually:
mean = torch.mean(embedding[0, :, :])
std = torch.std(embedding[0, :, :])
print((embedding[0, 0, :] - mean) / std)

which gives tensor([ 0.4310, -0.0319,  0.6523,  0.6050]) and that's not the right output. I want to know what is the right way to compute y[0, 0, :]?

Output:
Pytorch layer norm states mean and std calculated over last D dimensions. Based on this as I expect for (batch_size, seq_size, embedding_dim) here calculation should be over (seq_size, embedding_dim) for layer norm as last 2 dimensions excluding batch dim.
A similar question and answer with layer norm implementation can be found here, layer Normalization in pytorch?.
In some paper below it shows different layer norm application in NLP.
Explanation of Intance vs Layer vs Group Norm

From group norm paper

Layer Normalization (LN) operates along the channel dimension

LN computes µ and σ along the (C, H, W)
axes for each sample.

Different Application Example
In pytorch doc for NLP 3d tensor example mean and std instead are calculated over only last dim embedding_dim.
In this paper it shows similar to pytorch doc example,

almost all NLP tasks take variable length sequences as input, which is very suitable for LN
that only calculates statistics in the channel dimension without involving the batch and sequence length dimension.

Example shown in Another paper,

LN normalizes across the channel/feature dimension as shown in Figure 1.

Manual Layer Norm with only Embed Dim
import torch

batch_size, seq_size, dim = 2, 3, 4
last_dims = 4

embedding = torch.randn(batch_size, seq_size, dim)
print("x: ", embedding)

layer_norm = torch.nn.LayerNorm(last_dims, elementwise_affine = False)
layer_norm_out = layer_norm(embedding)
print("y: ", layer_norm_out)

eps: float = 0.00001
mean = torch.mean(embedding[0, :, :], dim=(-1), keepdim=True)
var = torch.square(embedding[0, :, :] - mean).mean(dim=(-1), keepdim=True)
y_custom = (embedding[0, :, :] - mean) / torch.sqrt(var + eps)
print("y_custom: ", y_custom)
assert torch.allclose(layer_norm_out[0], y_custom), 'Tensors do not match.'

eps: float = 0.00001
mean = torch.mean(embedding[1, :, :], dim=(-1), keepdim=True)
var = torch.square(embedding[1, :, :] - mean).mean(dim=(-1), keepdim=True)
y_custom = (embedding[1, :, :] - mean) / torch.sqrt(var + eps)
print("y_custom: ", y_custom)
assert torch.allclose(layer_norm_out[1], y_custom), 'Tensors do not match.'

Output
x:  tensor([[[-0.0594, -0.8702, -1.9837,  0.2914],
         [-0.4774,  1.0372,  0.6425, -1.1357],
         [ 0.3872, -0.9190, -0.5774,  0.3281]],

        [[-0.5548,  0.0815,  0.2333,  0.3569],
         [ 1.0380, -0.1756, -0.7417,  2.2930],
         [-0.0075, -0.3623,  1.9310, -0.7043]]])
y:  tensor([[[ 0.6813, -0.2454, -1.5180,  1.0822],
         [-0.5700,  1.1774,  0.7220, -1.3295],
         [ 1.0285, -1.2779, -0.6747,  0.9241]],

        [[-1.6638,  0.1490,  0.5814,  0.9334],
         [ 0.3720, -0.6668, -1.1513,  1.4462],
         [-0.2171, -0.5644,  1.6809, -0.8994]]])
y_custom:  tensor([[ 0.6813, -0.2454, -1.5180,  1.0822],
        [-0.5700,  1.1774,  0.7220, -1.3295],
        [ 1.0285, -1.2779, -0.6747,  0.9241]])
y_custom:  tensor([[-1.6638,  0.1490,  0.5814,  0.9334],
        [ 0.3720, -0.6668, -1.1513,  1.4462],
        [-0.2171, -0.5644,  1.6809, -0.8994]])

Manual Layer Norm over 4D Tensor
import torch

batch_size, c, h, w = 2, 3, 2, 4
last_dims = [c, h, w]

embedding = torch.randn(batch_size, c, h, w)
print("x: ", embedding)

layer_norm = torch.nn.LayerNorm(last_dims, elementwise_affine = False)
layer_norm_out = layer_norm(embedding)
print("y: ", layer_norm_out)

eps: float = 0.00001
mean = torch.mean(embedding[0, :, :], dim=(-3, -2, -1), keepdim=True)
var = torch.square(embedding[0, :, :] - mean).mean(dim=(-3, -2, -1), keepdim=True)
y_custom = (embedding[0, :, :] - mean) / torch.sqrt(var + eps)
print("y_custom: ", y_custom)
assert torch.allclose(layer_norm_out[0], y_custom), 'Tensors do not match.'

eps: float = 0.00001
mean = torch.mean(embedding[1, :, :], dim=(-3, -2, -1), keepdim=True)
var = torch.square(embedding[1, :, :] - mean).mean(dim=(-3, -2, -1), keepdim=True)
y_custom = (embedding[1, :, :] - mean) / torch.sqrt(var + eps)
print("y_custom: ", y_custom)
assert torch.allclose(layer_norm_out[1], y_custom), 'Tensors do not match.'

Output
x:  tensor([[[[ 1.0902, -0.8648,  1.5785,  0.3087],
          [ 0.0249, -1.3477, -0.9565, -1.5024]],

         [[ 1.8024, -0.2894,  0.7284,  0.7822],
          [ 1.4385, -0.2848, -0.3114,  0.4633]],

         [[ 0.9061,  0.3066,  0.9916,  0.9284],
          [ 0.3356,  0.9162, -0.4579,  1.0669]]],

        [[[-0.8292,  0.9111, -0.7307, -1.1003],
          [ 0.3441, -1.9823,  0.1313,  0.2048]],

         [[-0.2838,  0.1147, -0.1605, -0.4637],
          [-2.1343, -0.4402,  1.6685,  0.4455]],

         [[ 0.6895, -2.7331,  1.1693, -0.6999],
          [-0.3497, -0.2942, -0.0028, -1.3541]]]])
y:  tensor([[[[ 0.8653, -1.3279,  1.4131, -0.0114],
          [-0.3298, -1.8697, -1.4309, -2.0433]],

         [[ 1.6643, -0.6824,  0.4594,  0.5198],
          [ 1.2560, -0.6772, -0.7071,  0.1619]],

         [[ 0.6587, -0.0137,  0.7547,  0.6838],
          [ 0.0188,  0.6701, -0.8715,  0.8392]]],

        [[[-0.4938,  1.2220, -0.3967, -0.7610],
          [ 0.6629, -1.6306,  0.4531,  0.5256]],

         [[ 0.0439,  0.4368,  0.1655, -0.1335],
          [-1.7805, -0.1103,  1.9686,  0.7629]],

         [[ 1.0035, -2.3707,  1.4764, -0.3663],
          [-0.0211,  0.0337,  0.3210, -1.0112]]]])
y_custom:  tensor([[[ 0.8653, -1.3279,  1.4131, -0.0114],
         [-0.3298, -1.8697, -1.4309, -2.0433]],

        [[ 1.6643, -0.6824,  0.4594,  0.5198],
         [ 1.2560, -0.6772, -0.7071,  0.1619]],

        [[ 0.6587, -0.0137,  0.7547,  0.6838],
         [ 0.0188,  0.6701, -0.8715,  0.8392]]])
y_custom:  tensor([[[-0.4938,  1.2220, -0.3967, -0.7610],
         [ 0.6629, -1.6306,  0.4531,  0.5256]],

        [[ 0.0439,  0.4368,  0.1655, -0.1335],
         [-1.7805, -0.1103,  1.9686,  0.7629]],

        [[ 1.0035, -2.3707,  1.4764, -0.3663],
         [-0.0211,  0.0337,  0.3210, -1.0112]]])

Example of custom layer norm implementation
from typing import Union, List

import torch

batch_size, seq_size, embed_dim = 2, 3, 4
embedding = torch.randn(batch_size, seq_size, embed_dim)
print("x: ", embedding)
print(embedding.shape)
print()

layer_norm = torch.nn.LayerNorm(embed_dim, elementwise_affine=False)
layer_norm_output = layer_norm(embedding)
print("y: ", layer_norm_output)
print(layer_norm_output.shape)
print()

def custom_layer_norm(
        x: torch.Tensor, dim: Union[int, List[int]] = -1, eps: float = 0.00001
) -> torch.Tensor:
    mean = torch.mean(x, dim=(dim,), keepdim=True)
    var = torch.square(x - mean).mean(dim=(dim,), keepdim=True)
    return (x - mean) / torch.sqrt(var + eps)

custom_layer_norm_output = custom_layer_norm(embedding)
print("y_custom: ", custom_layer_norm_output)
print(custom_layer_norm_output.shape)

assert torch.allclose(layer_norm_output, custom_layer_norm_output), 'Tensors do not match.'

Output
x:  tensor([[[-0.4808, -0.1981,  0.4538, -1.2653],
         [ 0.3578,  0.6592,  0.2161,  0.3852],
         [ 1.2184, -0.4238, -0.3415, -0.3487]],

        [[ 0.9874, -1.7737,  0.1886,  0.0448],
         [-0.5162,  0.7872, -0.3433, -0.3266],
         [-0.5459, -0.0371,  1.2625, -1.6030]]])
torch.Size([2, 3, 4])

y:  tensor([[[-0.1755,  0.2829,  1.3397, -1.4471],
         [-0.2916,  1.5871, -1.1747, -0.1208],
         [ 1.7301, -0.6528, -0.5334, -0.5439]],

        [[ 1.1142, -1.6189,  0.3235,  0.1812],
         [-0.8048,  1.7141, -0.4709, -0.4384],
         [-0.3057,  0.1880,  1.4489, -1.3312]]])
torch.Size([2, 3, 4])

y_custom:  tensor([[[-0.1755,  0.2829,  1.3397, -1.4471],
         [-0.2916,  1.5871, -1.1747, -0.1208],
         [ 1.7301, -0.6528, -0.5334, -0.5439]],

        [[ 1.1142, -1.6189,  0.3235,  0.1812],
         [-0.8048,  1.7141, -0.4709, -0.4384],
         [-0.3057,  0.1880,  1.4489, -1.3312]]])
torch.Size([2, 3, 4])


Instruction: Best practice for controlling Pytorch's neural network layers' number and size?

Input:
I'm looking for best practices for controlling/adjusting the number of layers and also their sizes in Pytorch neural networks in general.
I have a configuration file in which I specify values for particular experiment variables. Additionally, I'd like to have an option in this file to determine the number and size of Pytorch's neural networks layers.
Current solution:
config.py
ACTOR_LAYER_SIZES: (128, 256, 128)

network.py
# input_size: int
# output_size: int
# layer_sizes = ACTOR_LAYER_SIZES

    layers = [
        nn.Linear(input_size, layer_sizes[0]),
        nn.ReLU(),
        BatchNorm1d(layer_sizes[0]),
    ]
    layers += list(
        chain.from_iterable(
            [
                [
                    nn.Linear(n_size, next_n_size),
                    nn.ReLU(),
                    BatchNorm1d(next_n_size),
                ]
                for n_size, next_n_size in zip(layer_sizes, layer_sizes[1:])
            ]
        )
    )
    layers += [(nn.Linear(layer_sizes[-1], output_size))]

    network = nn.Sequential(*layers)

I wonder if using chain.from_iterable may be considered here as the best practice in general. Furthermore, this code appears to be a little lengthy. Maybe there is a better way to do that?

Output:
I use the following snippet for this task:
import torch.nn as nn

num_inputs = 10
num_outputs = 5
hidden_layers = (128, 256, 128)
activation = nn.ReLU

layers = (
    num_inputs,
    *hidden_layers,
    num_outputs
)

network_architecture = []
for i in range(1, len(layers)):
    network_architecture.append(nn.Linear(layers[i - 1], layers[i]))
    if i < len(layers) - 1:  # for regression tasks prevent the last layer from getting an activation function
        network_architecture.append(activation())

model = nn.Sequential(*network_architecture)

The if statement prevents the output layer from getting an activation function. This is necessary when you do regression. If you want to do classification, however, you need some kind of activation function (e.g. softmax) there to output discrete classes.
Using a for loop in conjunction with an if statement instead of chain.from_iterable has the advantage that it is universally and intuitively understood. Furthermore, by moving the activation function out of the loop, it is configurable.
Adding the BatchNorm1d layer should be straightforward.


Instruction: Couldn't load the Pytorch optimised model in android

Input:
This is the error I'm getting in android while loading the model.
java.lang.RuntimeException: Unable to start activity ComponentInfo{com.android.example.cataractdetectionapp/com.android.example.cataractdetectionapp.InferenceActivity}:

com.facebook.jni.CppException: Could not run 'aten::empty_strided' with arguments from the 'CUDA' backend.
This could be because the operator doesn't exist for this backend, or was omitted during the selective/custom build process (if using custom build).
If you are a Facebook employee using PyTorch on mobile, please visit https://fburl.com/ptmfixes for possible resolutions.

'aten::empty_strided' is only available for these backends:
[CPU, Vulkan, BackendSelect, Python, Named, Conjugate, Negative, ADInplaceOrView, AutogradOther, AutogradCPU, AutogradCUDA,
AutogradXLA, AutogradLazy, AutogradXPU, AutogradMLC, UNKNOWN_TENSOR_TYPE_ID, Autocast, Batched, VmapMode].

CPU: registered at aten/src/ATen/RegisterCPU.cpp:18433 [kernel]

Vulkan: registered at ../aten/src/ATen/native/vulkan/ops/Factory.cpp:47 [kernel]
BackendSelect: registered at aten/src/ATen/RegisterBackendSelect.cpp:665 [kernel]
Python: registered at ../aten/src/ATen/core/PythonFallbackKernel.cpp:47 [backend fallback]
Named: registered at ../aten/src/ATen/core/NamedRegistrations.cpp:7 [backend fallback]
Conjugate: fallthrough registered at ../aten/src/ATen/ConjugateFallback.cpp:22 [kernel]
Negative: fallthrough registered at ../aten/src/ATen/native/NegateFallback.cpp:22 [kernel]
ADInplaceOrView: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:64 [backend fallback]
AutogradOther: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:35 [backend fallback]
AutogradCPU: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:39 [backend fallback]
AutogradCUDA: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:47 [backend fallback]
AutogradXLA: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:51 [backend fallback]
AutogradLazy: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:55 [backend fallback]
AutogradXPU: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:43 [backend fallback]
AutogradMLC: fallthrough registered at ../aten/src/ATen/core/VariableFallbackKernel.cpp:59 [backend fallback]
UNKNOWN_TENSOR_TYPE_ID: fallthrough registered at ../aten/src/ATen/autocast_mode.cpp:466 [backend fallback]
Autocast: fallthrough registered at ../aten/src/ATen/autocast_mode.cpp:305 [backend fallback]
Batched: registered at ../aten/src/ATen/BatchingRegistrations.cpp:1016 [backend fallback]
VmapMode: fallthrough registered at ../aten/src/ATen/VmapModeRegistrations.cpp:33 [backend fallback]

I used the below code for loading the module:
Module module = LiteModuleLoader.load(assetFilePath(getApplicationContext(), "Model.ptl"));

The dependencies I'm using for Pytorch are as follows:
implementation 'org.pytorch:pytorch_android_lite:1.10.0'
implementation 'org.pytorch:pytorch_android_torchvision:1.10.0'

Output: The solution to this error was simple. The error was because the model was saved with the Runtime being GPU. So, I loaded the native PyTorch model again in the CPU environment and saved it for Lite Interpreter and it is successfully loading in the android app.


Instruction: Dicom data training failed by pytorch

Input:
I've got a problem about training the Pytorch models. I'm trying to train my Pytorch model using dicom data and nifti GT However, the size of the weight file is ridiculously small because model training is not performed normally.
I used network model Unet++
I think there is a problem with the data loader. But I can't fixe it...
I'd appreciate it if you could help me.
Raw image file format is dicom and GT image format is nifti
in my dataloder
def __getitem__(self, index):
    image_path = self.image_paths[index]
    image_GT_path = image_path[:8]+'_'+image_path[8:12]+'.nii'
    GT_path = self.GT_paths + image_GT_path

    ds = dcmread(self.root+image_path)
    image = ds.pixel_array.astype(np.float32)
    image = torch.from_numpy(image.transpose(0,1)/255)
    image = image.unsqueeze(0)

    GT = nib.load(GT_path)
    GT = GT.get_fdata(dtype=np.float32)
    print(GT.shape)
    GT = torch.from_numpy(GT.transpose(0,1))
    GT = GT.unsqueeze(0)

    return image, GT, image_path

and Train Code is
for epoch in range(self.num_epochs):

            self.unet.train(True)
            epoch_loss = 0
            for i, (images, GT,empty) in enumerate(tqdm(self.train_loader)):
                # GT : Ground Truth

                images = images.to(self.device)
                GT = GT.to(self.device)

                # SR : Segmentation Result
                SR = self.unet(images)
                SR_probs = torch.sigmoid(SR)

                SR_flat = SR_probs.view(SR_probs.size(0),-1)

                GT_flat = GT.view(GT.size(0),-1)
                loss =self.criterion(SR_flat,GT_flat)

                # self.criterion=DiceLoss() #BCE not use
                # loss = self.criterion(GT,SR_probs)
                epoch_loss += loss.item()
                train_losses.append(loss.item())
                # Backprop + optimize
                self.reset_grad()
                loss.backward()
                self.optimizer.step()

Output:
Depending on what modality your images are, this might possibly be due to not converting the image data into the correct, clinically relevent, machine/vendor independent, units prior to any ML training 0-1 normalization.
Typically in dicom files, the actual raw data values aren't that - they need processing...
For instance, if you're trying to train on CT data, then the units you should be trying to train your model on are Houndsfield's (HU) numbers.  (Do a google on that, CT and dicom to get some background).
However raw CT dicom data could be little or big endian, likely needs a slope/intercept correction applied and also could need to have look up tables applied to convert it into HU numbers.  ...ie can get complicated and messy.  (again do a bit of googling ...you at least should have a bit of background on this if you're trying to do anything with medical image formats).
I'm not sure how to process nifti data, however luckily for dicom files using pydicom this conversion can be done for you by the library, using (typically) a call to pydicom.pixel_data_handlers.util.apply_modality_lut:

dcm = pydicom.dcmread(my_ct_dicom_file)
data_in_HU = pydicom.pixel_data_handlers.util.apply_voi_lut(
    dcm.pixel_array,
    dcm
)


Instruction: Does albumentations normalize mask?

Input:
When I pass an image and a mask to albumentations.Normalize(mean, std).
How would I go about incorporating this?
Should I just add it manually in dataset?
Grateful for any tips you have!

Output:
Edited:
Normalization works for three-channel images. If your mask image is grayscale image then probably you need to stack(image= np.stack((img,)*3, axis=-1))
it and make three channel image then apply albumentations's Normalization function. Official function for A.Normalize() is as following which deals with RGB images:
def normalize(img, mean, std, max_pixel_value=255.0):
    mean = np.array(mean, dtype=np.float32)
    mean *= max_pixel_value

    std = np.array(std, dtype=np.float32)
    std *= max_pixel_value

    denominator = np.reciprocal(std, dtype=np.float32)

    img = img.astype(np.float32)
    img -= mean
    img *= denominator
    return img

According to Albumentations's docs, you can make a composition of Transforms and use it within PyTorch dataset.
import albumentations as A
from albumentations.pytorch import ToTensorV2

train_transform = A.Compose(
    [
        A.SmallestMaxSize(max_size=160),
        A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),
        A.RandomCrop(height=128, width=128),
        A.RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5),
        A.RandomBrightnessContrast(p=0.5),
        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
        ToTensorV2(),
    ]
)
train_dataset = CatsVsDogsDataset(images_filepaths=train_images_filepaths, transform=train_transform)

But I am not really sure that normalizing mask image is right way or not.


Instruction: NameError: name '_C' is not defined

Input:
When running a Python project using torch==1.4.0 I got the error in the title. I am using Linux and not using any kind of IDE or notebook.
There are other questions about this, but they were all in Jupyter notebooks or on Windows.

Output:
What happened is that the version of libffi I was using was too new. It looks like libffi recently upgraded to version 8, but something (Torch?) required v7. v7 not being present caused some kind of import to fail silently rather than given a clear error, resulting in the error in the title.
I was able to fix this by installing libffi7 using my operating system's package manager.


Instruction: How can I change self attention layer numbers and multihead attention head numbers in my model with Pytorch?

Input:
I working on sarcasm dataset and my model like below:
I first tokenize my input text:
 PRETRAINED_MODEL_NAME = "roberta-base"
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)
import torch
from torch.utils.data import Dataset, DataLoader

MAX_LEN = 100

then I defined class for my dataset:
class SentimentDataset (Dataset):
    def __init__(self,dataframe):
        self.dataframe = dataframe

    def __len__(self):
        return len(self.dataframe)

    def __getitem__(self, idx):
        df = self.dataframe.iloc[idx]

        text = [df["comment"]]
        label = [df["label"]]

        data_t = tokenizer(text,max_length = MAX_LEN, return_tensors="pt", padding="max_length", truncation=True)
        label_t = torch.LongTensor(label)

        return {
             "input_ids":data_t["input_ids"].squeeze().to(device),
             "label": label_t.squeeze().to(device),
        }

then I create obj from my class for training set and set other parameters:
train_dataset = SentimentDataset(train_df)
BATCH_SIZE = 32
train_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE)
from transformers import AutoModelForSequenceClassification, AutoConfig

# For loading model stucture and pretrained weights:
model = AutoModelForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME).to(device)

import transformers

optimizer = torch.optim.Adam(model.parameters(), lr=2e-5, weight_decay=1e-5)

Then I use dataloader for training my data:
train_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE)
EPOCHS = 5
for epoch in range(EPOCHS):
    print("\n******************\n epoch=",epoch)
    i = 0
    logits_list = []
    labels_list = []
    for batch in train_dataloader:
        i += 1
        optimizer.zero_grad()
        output_model = model(input_ids = batch["input_ids"], labels = batch["label"])
        loss = output_model.loss
        logits = output_model.logits
        logits_list.append(logits.cpu().detach().numpy())
        labels_list.append(batch["label"].cpu().detach().numpy())
        loss.backward()
        optimizer.step()
    #scheduler.step()
        if i % 50 ==0 :
            print("training loss:",loss.item())
            #print("validation loss:",loss.item())
    logits_list = np.concatenate(logits_list, axis=0)
    labels_list = np.concatenate(labels_list, axis=0)
    logits_list = np.argmax(logits_list, axis =1)
    print(classification_report(labels_list, logits_list))

My question is how can I change self attention layers number and head of multihead attention in my model?

Output:
The short answer is: You can't.
You are using a pretrained model:
model = AutoModelForSequenceClassification.from_pretrained(PRETRAINED_MODEL_NAME).to(device)

You can't easily change the pretrained model. It is possible to change pretrained models, but that is definitely not straightforward. You can download different pretrained model or you can train any model you like from scratch (which would take probably too much time and computational resources). The only thing you can easily change is the "depth" of the model - you can discard some of the transformer blocks.


Instruction: Failed to install Pytorch Snippets and Librosa in VScode - Apple M1

Input:
I anticipate I'm not an expert in informatics. I used to run PyTorch snippets for some deep learning on my old MacBook Pro (2015), but now I have Apple's last Pro, and have troubles with installing packages in VScode. Two of these packages give me trouble: PyTorch snippets and Librosa. PyTorch Snippets can be installed from the "extensions" in the VScode menu; that is what I did, and used to do with my old Mac. This time, however, when I import torch within an .ipynb instance I get the error "No module named 'torch'".
With Librosa, when I run in the terminal pip3 install librosa at some point I get this error:
Using cached idna-3.3-py3-none-any.whl (61 kB)
Building wheels for collected packages: numba, llvmlite
Building wheel for numba (setup.py) ... error
, then a long list of red things, then again:
ERROR: Failed building wheel for llvmlite
Running setup.py clean for llvmlite
Failed to build numba llvmlite
Installing collected packages: llvmlite, idna, charset-normalizer, certifi, threadpoolctl, scipy, requests, numba, joblib, appdirs, soundfile, scikit-learn, resampy, pooch, audioread, librosa
Running setup.py install for llvmlite ... error
ERROR: Command errored out with exit status 1:
Do you experienced anything similar? Do you have some hint? Thanks a lot

Output:
There is a fix described in
https://github.com/numba/llvmlite/issues/693#issuecomment-909501195
arch -arm64 brew install llvm@11
LLVM_CONFIG="/opt/homebrew/Cellar/llvm@11/11.1.0_2/bin/llvm-config" arch -arm64 pip install llvmlite


Instruction: why and when to use torch.cuda.Stream()

Input:
I found torch.cuda.Stream() is manually defined in some open source code.
    self.input_stream = torch.cuda.Stream()
    self.model_stream = torch.cuda.Stream()
    self.output_stream = torch.cuda.Stream()

On torch page, it says
You normally do not need to create one explicitly: by default, each device uses its own “default” stream.
Trying to understand why they had to define this manually. From the quick google search, there are lots of how to use cuda.Stream() but no why/when/best-practice to use it.

Output: Streams are sequences of cuda kernels. Operations in different streams may run in parallel. I don't believe they have to use them. They are are just making the code more parallel and thus hopefully faster.


Instruction: How to concatenate a 0-D tensor into a 3-D tensor in PyTorch?

Input:
I have the following tensor:
X = torch.randn(30,1,2)  # [batch_size, dim_1, dim_2]

t = torch.Tensor([0])

I am trying to concatenate the t tensor into X tensor that results [30,1,3] tensor. However, I tried couple of methods even with torch.stack. I still have not figured out how to do this properly. I tried both and they gave errors.
result = torch.cat((X,t), dim = -1) # first try
result = torch.stack([X,t], dim = -1) # second try.

Is there a way I can concatenate these tensors?

Output:
You can't concatenate the two described tensors, the shape of tensor X is [30, 1 , 2], which means it has 30 positions in the first dimension, 1 position in the second dimension, and 2 positions in the last dimension, totalling 30*1*2 = 60 elements. A tensor of shape [30,1,3] has 90 elements, meaning you need to add 30 elements to get the desired result.
You can do this by changing the code to:
>>> X = torch.randn(30,1,2)
>>> t = torch.zeros(30,1,1)
>>> r = torch.cat((X,t), dim=-1)
>>> r.shape
torch.Size([30, 1, 3])


Instruction: PyTorch Lightning auto_scale_batch_size='power' does not show results

Input:
I am very new to Deep Learning, and am converting an existing project into a Pytorch Lightning one following this tutorial.
I want to try the automatic batch size finder. So I added the requested flag to the Trainer :
trainer = pl.Trainer(default_root_dir=model_dir,
                     auto_scale_batch_size='power')

and also added a batch_size parameter to the model's init method that is then used with self.batch_size = batch_size.
class MyModel(pl.LightningModule):
  def __init__(self, other params, batch_size)
     super().__init__()
     self.batch_size = batch_size

In train_dataloader I also use this self.batch_size :
train_loader = DataLoader(self.dataset,
                             collate_fn=lambda batch: collate_synthesizer(batch, self.reduction_factor, self.hparams2),
                             batch_size=self.batch_size,
                             num_workers=self.num_workers if platform.system() != "Windows" else 0,
                             shuffle=True,
                             pin_memory=True)

But the training process runs without showing anything special about its findings with regards to batch_size. I tried it on my dev laptop (cpu mode) and on colab (gpu) without seeing anything.
Should I wait till the end of the process or I missed something ?
Thanks you very much

Output: My bad, I missed something in the video that I later found in the doc. I was calling trainer.fit(model) instead of trainer.tune(model). Now it is working great!


Instruction: Problem encountered in MMDetection. KeyError: 'mask_detectionDataset is not in the dataset registry'

Input:
I tried to train my model with MMdetection, however, error like "KeyError: 'mask_detectionDataset is not in the dataset registry'" keep showing.
I've added my dataset to init.py in \mmdetection\mmdet\datasets. And use @DATASETS.register_module(), but problem doesn't solved.
When I try to run init.py directly in \mmdetection\mmdet\datasets, it shows attempted relative import with no known parent package,i'm wondering why.
here is my code:
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 27 00:55:00 2021

@author: daish
"""
import mmcv
from mmcv import Config
from mmdet.apis import set_random_seed
import os

cfg = Config.fromfile('F:/Project/mmdetection/configs/swin/mask_rcnn_swin-t-p4-w7_fpn_1x_coco.py')
# Modify dataset type and path
cfg.dataset_type = 'mask_detectionDataset'
cfg.data_root = 'F:/Project/dataset/'

cfg.data.test.type = 'mask_detectionDataset'
cfg.data.test.data_root = 'F:/Project/dataset/'
cfg.data.test.ann_file = 'test.txt'
cfg.data.test.img_prefix = 'images'

cfg.data.train.type = 'mask_detectionDataset'
cfg.data.train.data_root = 'F:/Project/dataset/'
cfg.data.train.ann_file = 'train.txt'
cfg.data.train.img_prefix = 'images'

cfg.data.val.type = 'mask_detectionDataset'
cfg.data.val.data_root = 'F:/Project/dataset/'
cfg.data.val.ann_file = 'val.txt'
cfg.data.val.img_prefix = 'images'

# modify num classes of the model in box head
cfg.model.roi_head.bbox_head.num_classes = 3
cfg.model.roi_head.mask_head.num_classes = 3
# We can still use the pre-trained Mask RCNN model though we do not need to
# use the mask branch
cfg.load_from = 'https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_tiny_patch4_window7_224.pth'

# Set up working dir to save files and logs.
cfg.work_dir = './swin/mask_rcnn_swin-t-p4-w7_fpn_1x'

# The original learning rate (LR) is set for 8-GPU training.
# We divide it by 8 since we only use one GPU.
cfg.optimizer.lr = 0.02 / 8
cfg.lr_config.warmup = None
cfg.log_config.interval = 10

# Change the evaluation metric since we use customized dataset.
cfg.evaluation.metric = 'mAP'
# We can set the evaluation interval to reduce the evaluation times
cfg.evaluation.interval = 12
# We can set the checkpoint saving interval to reduce the storage cost
cfg.checkpoint_config.interval = 12

# Set seed thus the results are more reproducible
cfg.seed = 0
set_random_seed(0, deterministic=False)
cfg.gpu_ids = range(1)

# We can initialize the logger for training and have a look
# at the final config used for training
print(f'Config:\n{cfg.pretty_text}')

from mmdet.datasets import build_dataset
from mmdet.models import build_detector
from mmdet.apis import train_detector

# Build dataset
datasets = [build_dataset(cfg.data.train)]

# Build the detector
model = build_detector(
    cfg.model, train_cfg=cfg.get('train_cfg'), test_cfg=cfg.get('test_cfg'))
# Add an attribute for visualization convenience
model.CLASSES = datasets[0].CLASSES

# Create work_dir
mmcv.mkdir_or_exist(os.path.abspath(cfg.work_dir))
train_detector(model, datasets, cfg, distributed=False, validate=True)

below is the error:
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "D:\anaconda3\envs\openmmlab\lib\multiprocessing\spawn.py", line 105, in spawn_main
    exitcode = _main(fd)
  File "D:\anaconda3\envs\openmmlab\lib\multiprocessing\spawn.py", line 114, in _main
    prepare(preparation_data)
  File "D:\anaconda3\envs\openmmlab\lib\multiprocessing\spawn.py", line 225, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "D:\anaconda3\envs\openmmlab\lib\multiprocessing\spawn.py", line 277, in _fixup_main_from_path
    run_name="__mp_main__")
  File "D:\anaconda3\envs\openmmlab\lib\runpy.py", line 263, in run_path
    pkg_name=pkg_name, script_name=fname)
  File "D:\anaconda3\envs\openmmlab\lib\runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "D:\anaconda3\envs\openmmlab\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "F:\Project\config.py", line 70, in <module>
    datasets = [build_dataset(cfg.data.train)]
  File "D:\anaconda3\envs\openmmlab\lib\site-packages\mmdet\datasets\builder.py", line 80, in build_dataset
    dataset = build_from_cfg(cfg, DATASETS, default_args)
  File "D:\anaconda3\envs\openmmlab\lib\site-packages\mmcv\utils\registry.py", line 44, in build_from_cfg
    f'{obj_type} is not in the {registry.name} registry')
KeyError: 'mask_detectionDataset is not in the dataset registry'

Output:
maybe add a custom_imports key in your config?
custom_imports = dict(
    imports=['mmdet.datasets.mask_detectionDataset'],
    allow_failed_imports=False)