Instruction: How to calculate the f1-score?

Input:
I have a pyTorch-code to train a model that should be able to detect placeholder-images among product-images. I didn't write the code by myself as I am very unexperienced with CNNs and Machine Learning.
My boss told me to calculate the f1-score for that model and i found out that the formula for that is ((precision * recall)/(precision + recall)) but I don't know how I get precision and recall. Is someone able to tell me how I can get those two parameters from that following code?
(Sorry for the long piece of code, but I didn't really know what is necessary and what isn't)
from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
print("PyTorch Version: ",torch.__version__)
print("Torchvision Version: ",torchvision.__version__)

data_dir = "data"

# Models to choose from [resnet, alexnet, vgg, squeezenet, densenet, inception]
model_name = "resnet"

# Number of classes in the dataset [we have four classes A-Balik-Duz-Princess]
num_classes = 2

# Batch size for training (change depending on how much memory you have)
batch_size = 25

# Number of epochs to train for (This will need to be calculated in order to address under and over fitting issue)
num_epochs = 20

# Flag for feature extracting. When False, we fine tune the whole model,
#   when True we only update the reshaped layer params
feature_extract = True

def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):
    since = time.time()
    print("model is : ",model)

    val_acc_history = []
    val_loss_history = []
    train_acc_history = []
    train_loss_history = []
    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients (This can be changed to the Adam and other optimizers)
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    # Get model outputs and calculate loss
                    # Special case for inception because in training it has an auxiliary output. In train
                    #   mode we calculate the loss by summing the final output and the auxiliary output
                    #   but in testing we only consider the final output.
                    if is_inception and phase == 'train':
                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958
                        outputs, aux_outputs = model(inputs)
                        loss1 = criterion(outputs, labels)
                        loss2 = criterion(aux_outputs, labels)
                        loss = loss1 + 0.4*loss2
                    else:
                        outputs = model(inputs)
                        loss = criterion(outputs, labels)

                    _, preds = torch.max(outputs, 1)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
            if phase == 'val':
                val_acc_history.append(epoch_acc)
                val_loss_history.append(epoch_loss)
            if phase == 'train':
                train_acc_history.append(epoch_acc)
                train_loss_history.append(epoch_loss)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history, train_acc_history,val_loss_history,train_loss_history

def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False

###############################################
###   Initialize and Reshape the Networks
###############################################

def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
    # Initialize these variables which will be set in this if statement. Each of these
    #   variables is model specific.
    model_ft = None
    input_size = 0

    if model_name == "resnet":
        """ Resnet18
        """
        model_ft = models.resnet152(pretrained=use_pretrained)
        #we can select any possible variation of ResNet such as Resnet18, Resnet34, Resnet50, Resnet101, and Resnet152
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "alexnet":
        """ Alexnet
        """
        model_ft = models.alexnet(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier[6].in_features
        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
        input_size = 224

    elif model_name == "vgg":
        """ VGG11_bn
        """
        model_ft = models.vgg11_bn(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier[6].in_features
        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
        input_size = 224

    elif model_name == "squeezenet":
        """ Squeezenet
        """
        model_ft = models.squeezenet1_0(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))
        model_ft.num_classes = num_classes
        input_size = 224

    elif model_name == "densenet":
        """ Densenet
        """
        model_ft = models.densenet121(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier.in_features
        model_ft.classifier = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "inception":
        """ Inception v3
        Be careful, expects (299,299) sized images and has auxiliary output
        """
        model_ft = models.inception_v3(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        # Handle the auxilary net
        num_ftrs = model_ft.AuxLogits.fc.in_features
        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
        # Handle the primary net
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs,num_classes)
        input_size = 299

    else:
        print("Invalid model name, exiting...")
        exit()

    return model_ft, input_size

# Initialize the model for this run
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)

# Print the model we just instantiated
#print(model_ft)

########################
###   LOAD DATA
########################

# Data augmentation and normalization for training
# there are multiple approaches for data augmentation  which can be added in the future
# Just normalization for validation
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomResizedCrop(input_size),
        #transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(input_size),
        transforms.CenterCrop(input_size),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

print("Initializing Datasets and Dataloaders...")

# Create training and validation datasets
image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}
# Create training and validation dataloaders
dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}

# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

#############################
###   Create the Optimizer
#############################

# Send the model to GPU
model_ft = model_ft.to(device)

# Gather the parameters to be optimized/updated in this run. If we are
#  fine tuning we will be updating all parameters. However, if we are
#  doing feature extract method, we will only update the parameters
#  that we have just initialized, i.e. the parameters with requires_grad
#  is True.
params_to_update = model_ft.parameters()
print("Params to learn:")
if feature_extract:
    params_to_update = []
    for name,param in model_ft.named_parameters():
        if param.requires_grad == True:
            params_to_update.append(param)
            print("\t",name)
else:
    for name,param in model_ft.named_parameters():
        if param.requires_grad == True:
            print("\t",name)

# Observe that all parameters are being optimized we can add leaky ReLU and much more
optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)

###########################
###   Run Training and Validation Step
###########################

%time
# Setup the loss fxn
criterion = nn.CrossEntropyLoss()

# Train and evaluate
model_ft, hist, loss_t,vloss_acc, tloss_acc = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs, is_inception=(model_name=="inception"))

Output:
...
# statistics
    running_loss += loss.item() * inputs.size(0)
    running_corrects += torch.sum(preds == labels.data)
    # Add these lines to obtain f1_score
    from sklearn.metrics import f1_score
    f1_score = f1_score(labels.data, preds)
    #or: f1_score = f1_score(labels.cpu().data, preds.cpu())
...


Instruction: How can I get torch-geometric to work using Nix?

Input:
I am trying to get the Python package torch-geometric to work using Nix (I am on NixOS). Currently, I use mach-nix to try and setup a Python environment. However, the difficulty is that some of the dependencies should be downloaded from a separate file server (not pypi), i.e. https://pytorch-geometric.com/whl/torch-1.8.0+cpu.html. I am first trying to setup an environment containing a single torch-geometric dependency: torch-sparse.
Currently I have the following shell.nix:
{ pkgs ? import <nixpkgs> {} }:

let
  mach-nix = import (builtins.fetchGit {
    url = "https://github.com/DavHau/mach-nix/";
    ref = "refs/tags/3.3.0";
  }) {
    python = "python38";
  };
  sparse = mach-nix.buildPythonPackage {
    pname = "torch_sparse";
    version = "0.6.9";
    requirements = ''
      torch
      scipy
      pytest
      pytest-cov
      pytest-runner
    '';
    src = builtins.fetchGit {
      url = "https://github.com/rusty1s/pytorch_sparse";
      ref = "refs/tags/0.6.9";
    };
  };
in mach-nix.mkPython {
  requirements = "torch-sparse";
  packagesExtra = [
    sparse
  ];
}

Which, upon running nix-shell, fails with the following error message:
running build_ext
error: [Errno 2] No such file or directory: 'which'
builder for '/nix/store/fs9nrrd2a233xp5d6njy6639yjbxp4g0-python3.8-torch_sparse-0.6.9.drv' failed with exit code 1

I tried adding the which package to either checkInputs and buildInputs, but that does not solve the problem. Evidently, I try to build the package directly from its GitHub repo, as I am unsure on how to reference a wheel package in mach-nix. I am relatively new to the NixOS environment, and, quite frankly, I am completely lost.
How should I go about installing a Python package such as torch-sparse or torch-geometric? Am I even using the correct tools?

Output:
I have managed to come up with a working Nix expression. I will leave the answer here for future reference. Running the following expression using nix-shell will create a shell with torch-1.8.0 and torch-geometric-1.7.0 and their required dependencies.
{ pkgs ? import <nixpkgs> { } }:

let
  python = pkgs.python38;
  pytorch-180 = let
    pyVerNoDot = builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion;
    unsupported = throw "Unsupported system";
    version = "1.8.0";
  in python.pkgs.buildPythonPackage {
    inherit version;

    pname = "pytorch";

    format = "wheel";

    src = pkgs.fetchurl {
      name = "torch-${version}-cp38-cp38-linux_x86_64.whl";
      url =
        "https://download.pytorch.org/whl/cu111/torch-${version}%2Bcu111-cp38-cp38-linux_x86_64.whl";
      hash = "sha256-4NYiAkYfGXm3orLT8Y5diepRMAg+WzJelncy2zJp+Ho=";
    };

    nativeBuildInputs = with pkgs; [ addOpenGLRunpath patchelf ];

    propagatedBuildInputs = with python.pkgs; [
      future
      numpy
      pyyaml
      requests
      typing-extensions
    ];

    postInstall = ''
      # ONNX conversion
      rm -rf $out/bin
    '';

    postFixup = let rpath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ];
    in ''
      find $out/${python.sitePackages}/torch/lib -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do
        echo "setting rpath for $lib..."
        patchelf --set-rpath "${rpath}:$out/${python.sitePackages}/torch/lib" "$lib"
        addOpenGLRunpath "$lib"
      done
    '';

    pythonImportsCheck = [ "torch" ];

    meta = with pkgs.lib; {
      description =
        "Open source, prototype-to-production deep learning platform";
      homepage = "https://pytorch.org/";
      changelog = "https://github.com/pytorch/pytorch/releases/tag/v${version}";
      license = licenses.unfree; # Includes CUDA and Intel MKL.
      platforms = platforms.linux;
      maintainers = with maintainers; [ danieldk ];
    };
  };
  sparse = with python.pkgs;
    buildPythonPackage rec {
      pname = "torch_sparse";
      version = "0.6.9";

      src = pkgs.fetchurl {
        name = "${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        url =
          "https://pytorch-geometric.com/whl/torch-1.8.0+cpu/${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        hash = "sha256-6dmZNQ0FlwKdfESKhvv8PPwzgsJFWlP8tYXWu2JLiMk=";
      };

      format = "wheel";

      propagatedBuildInputs = [ pytorch-180 scipy ];
      # buildInputs = [ pybind11 ];
      # nativeBuildInputs = [ pytest-runner pkgs.which ];

      doCheck = false;

      postInstall = ''
        rm -rf $out/${python.sitePackages}/test
      '';
    };
  scatter = with python.pkgs;
    buildPythonPackage rec {
      pname = "torch_scatter";
      version = "2.0.7";

      src = pkgs.fetchurl {
        name = "${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        url =
          "https://pytorch-geometric.com/whl/torch-1.8.0+cpu/${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        hash = "sha256-MRoFretgyEpq+7aJZc0399Kd+f28Uhn5+CxW5ZIKwcg=";
      };

      format = "wheel";

      propagatedBuildInputs = [ pytorch-180 ];

      doCheck = false;

      postInstall = ''
        rm -rf $out/${python.sitePackages}/test
      '';
    };
  cluster = with python.pkgs;
    buildPythonPackage rec {
      pname = "torch_cluster";
      version = "1.5.9";

      src = pkgs.fetchurl {
        name = "${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        url =
          "https://pytorch-geometric.com/whl/torch-1.8.0+cpu/${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        hash = "sha256-E2nywtiZ7m7VA1J7AY7gAHYvyN9H3zl/W0/WsZLzwF8=";
      };

      format = "wheel";

      propagatedBuildInputs = [ pytorch-180 ];

      doCheck = false;

      postInstall = ''
        rm -rf $out/${python.sitePackages}/test
      '';
    };
  spline = with python.pkgs;
    buildPythonPackage rec {
      pname = "torch_spline_conv";
      version = "1.2.1";

      src = pkgs.fetchurl {
        name = "${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        url =
          "https://pytorch-geometric.com/whl/torch-1.8.0+cpu/${pname}-${version}-cp38-cp38-linux_x86_64.whl";
        hash = "sha256-ghSzoxoqSccPAZzfcHJEPYySQ/KYqQ90mFsOdt1CjUw=";
      };

      format = "wheel";

      propagatedBuildInputs = [ pytorch-180 ];

      doCheck = false;

      postInstall = ''
        rm -rf $out/${python.sitePackages}/test
      '';
    };
  python-louvain = with python.pkgs;
    buildPythonPackage rec {
      pname = "python-louvain";
      version = "0.15";

      src = fetchPypi {
        inherit pname version;
        sha256 = "1sqp97fwh4asx0jr72x8hil8z8fcg2xq92jklmh2m599pvgnx19a";
      };

      propagatedBuildInputs = [ numpy networkx ];

      doCheck = false;
    };
  googledrivedownloader = with python.pkgs;
    buildPythonPackage rec {
      pname = "googledrivedownloader";
      version = "0.4";

      src = fetchPypi {
        inherit pname version;
        sha256 = "0172l1f8ys0913wcr16lzx87vsnapppih62qswmvzwrggcrw2d2b";
      };

      doCheck = false;
    };
  geometric = with python.pkgs;
    buildPythonPackage rec {
      pname = "torch_geometric";
      version = "1.7.0";

      src = fetchPypi {
        inherit pname version;
        sha256 = "1a7ym34ynhk5gb3yc5v4qkmkrkyjbv1fgisrsk0c9xay66w7nwz9";
      };

      propagatedBuildInputs = [
        pytorch-180
        numpy
        scipy
        tqdm
        networkx
        scikit-learn
        requests
        pandas
        rdflib
        jinja2
        numba
        ase
        h5py
        python-louvain
        googledrivedownloader
      ];
      nativeBuildInputs = [ pytest-runner ];

      doCheck = false;

      # postInstall = ''
      #   rm -rf $out/${python.sitePackages}/test
      # '';
    };
  python-with-pkgs = python.withPackages
    (ps: with ps; [ pytorch-180 scatter sparse cluster spline geometric ps ]);
in pkgs.mkShell { buildInputs = [ python-with-pkgs ]; }


Instruction: Derivative of BatchNorm2d in PyTorch

Input:
In my network, I want to calculate the forward pass and backward pass of my network both in the forward pass.
For this, I have to manually define all the backward pass methods of the forward pass layers.
For the activation functions, that's easy. And also for the linear and conv layers, it worked well. But I'm really struggling with BatchNorm. As the BatchNorm paper only discusses the 1D case:
So far, my implementation looks like this:
def backward_batchnorm2d(input, output, grad_output, layer):
    gamma = layer.weight
    beta = layer.bias
    avg = layer.running_mean
    var = layer.running_var
    eps = layer.eps
    B = input.shape[0]

    # avg, var, gamma and beta are of shape [channel_size]
    # while input, output, grad_output are of shape [batch_size, channel_size, w, h]
    # for my calculations I have to reshape avg, var, gamma and beta to [batch_size, channel_size, w, h] by repeating the channel values over the whole image and batches

    dL_dxi_hat = grad_output * gamma
    dL_dvar = (-0.5 * dL_dxi_hat * (input - avg) / ((var + eps) ** 1.5)).sum((0, 2, 3), keepdim=True)
    dL_davg = (-1.0 / torch.sqrt(var + eps) * dL_dxi_hat).sum((0, 2, 3), keepdim=True) + dL_dvar * (-2.0 * (input - avg)).sum((0, 2, 3), keepdim=True) / B
    dL_dxi = dL_dxi_hat / torch.sqrt(var + eps) + 2.0 * dL_dvar * (input - avg) / B + dL_davg / B # dL_dxi_hat / sqrt()
    dL_dgamma = (grad_output * output).sum((0, 2, 3), keepdim=True)
    dL_dbeta = (grad_output).sum((0, 2, 3), keepdim=True)
    return dL_dxi, dL_dgamma, dL_dbeta

When I check my gradients with torch.autograd.grad() I notice that dL_dgamma and dL_dbeta are correct, but dL_dxi is incorrect,  (by a lot). But I can't find my mistake. Where is my mistake?
For reference, here is the definition of BatchNorm:

And here are the formulas for the derivatives for the 1D case:

Output:
def backward_batchnorm2d(input, output, grad_output, layer):
    gamma = layer.weight
    gamma = gamma.view(1,-1,1,1) # edit
    # beta = layer.bias
    # avg = layer.running_mean
    # var = layer.running_var
    eps = layer.eps
    B = input.shape[0] * input.shape[2] * input.shape[3] # edit

    # add new
    mean = input.mean(dim = (0,2,3), keepdim = True)
    variance = input.var(dim = (0,2,3), unbiased=False, keepdim = True)
    x_hat = (input - mean)/(torch.sqrt(variance + eps))

    dL_dxi_hat = grad_output * gamma
    # dL_dvar = (-0.5 * dL_dxi_hat * (input - avg) / ((var + eps) ** 1.5)).sum((0, 2, 3), keepdim=True)
    # dL_davg = (-1.0 / torch.sqrt(var + eps) * dL_dxi_hat).sum((0, 2, 3), keepdim=True) + dL_dvar * (-2.0 * (input - avg)).sum((0, 2, 3), keepdim=True) / B
    dL_dvar = (-0.5 * dL_dxi_hat * (input - mean)).sum((0, 2, 3), keepdim=True)  * ((variance + eps) ** -1.5) # edit
    dL_davg = (-1.0 / torch.sqrt(variance + eps) * dL_dxi_hat).sum((0, 2, 3), keepdim=True) + (dL_dvar * (-2.0 * (input - mean)).sum((0, 2, 3), keepdim=True) / B) #edit

    dL_dxi = (dL_dxi_hat / torch.sqrt(variance + eps)) + (2.0 * dL_dvar * (input - mean) / B) + (dL_davg / B) # dL_dxi_hat / sqrt()
    # dL_dgamma = (grad_output * output).sum((0, 2, 3), keepdim=True)
    dL_dgamma = (grad_output * x_hat).sum((0, 2, 3), keepdim=True) # edit
    dL_dbeta = (grad_output).sum((0, 2, 3), keepdim=True)
    return dL_dxi, dL_dgamma, dL_dbeta

Because you didn't upload your forward snipcode, so if your gamma has the shape size is 1, you need to reshape it to [1,gamma.shape[0],1,1].
The formula follows 1D where the scale factor is the sum of the batch size. However, in 2D, the summation should between 3 dimensions, so B = input.shape[0] * input.shape[2] * input.shape[3].
The running_mean and running_var only use in test/inference mode, we don't use them in training (you can find it in the paper). The mean and variance you need are computed from the input, you can store the mean, variance and x_hat = (x-mean)/sqrt(variance + eps) into your object layer or re-compute as I did in the code above # add new. Then replace them with the formula of dL_dvar, dL_davg, dL_dxi.
your dL_dgamma should be incorrect since you multiplied the gradient of output by itself, it should be modified to grad_output * x_hat.


Instruction: How to prevent gradient computations for certain elements of a tensor in Pytorch

Input:
To be clear, I am not

Asking how to prevent gradients from being propagated to certain tensors (in this case you can just set requires_grad = False for that tensor).
Asking how to prevent gradients from being propagated from an entire tensor (in that case you can just call tensor.detach(), see this question).

I'm wondering how to forgo gradient computations for some elements of a loss tensor that give a NaN gradient every time -- essentially, to call .detach() for individual elements of a tensor. The way to do this in Tensorflow is using tf.stop_gradients, see this question.
Some context: My neural network computes a distance matrix of its predicted coordinates, as follows. The entries of the distance matrix D are given by d_ij = || coordinates_i - coordinates_j ||. I want to backpropagate through the distance matrix creation step. However, the norm function includes a square root, which is not differentiable at 0 -- and the diagonal of the distance matrix is 0 by construction. Thus I get NaN gradients for the diagonal of the distance matrix. I would like to mask out the gradients on the diagonal of the distance matrix.
Minimal working example:
import torch

def compute_distance_matrix(coordinates):
    L = len(coordinates)
    gram_matrix = torch.mm(coordinates, torch.transpose(coordinates, 0, 1))
    gram_diag = torch.diagonal(gram_matrix, dim1=0, dim2=1)
    # gram_diag: L
    diag_1 = torch.matmul(gram_diag.unsqueeze(-1), torch.ones(1, L).to(coordinates.device))
    # diag_1: L x L
    diag_2 = torch.transpose(diag_1, dim0=0, dim1=1)
    # diag_2: L x L
    distance_matrix = torch.sqrt(diag_1 + diag_2 - (2 * gram_matrix))
    return distance_matrix

# In reality, pred_coordinates is an output of the network, but we initialize it here for a minimal working example
L = 10
pred_coordinates = torch.randn(L, 3, requires_grad=True)
true_coordinates = torch.randn(L, 3, requires_grad=False)
obj = torch.nn.MSELoss()
optimizer = torch.optim.Adam([pred_coordinates])

for i in range(500):
    pred_distance_matrix = compute_distance_matrix(pred_coordinates)
    true_distance_matrix = compute_distance_matrix(true_coordinates)
    loss = obj(pred_distance_matrix, true_distance_matrix)
    loss.backward()
    print(loss.item())
    optimizer.step()

gives
1.2868314981460571
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
nan
...

Output:
I initialized a new matrix and used a mask to copy the values with differentiable gradients from the previous tensor (in this case, the non-diagonal entries), then applied the not-everywhere-differentiable operation (the square root) to the new tensor. This allowed the gradient to only flow back through the entries that had a positive mask.
import torch

def compute_distance_matrix(coordinates):
    # In reality, pred_coordinates is an output of the network, but we initialize it here for a minimal working example
    L = len(coordinates)
    gram_matrix = torch.mm(coordinates, torch.transpose(coordinates, 0, 1))
    gram_diag = torch.diagonal(gram_matrix, dim1=0, dim2=1)
    # gram_diag: L
    diag_1 = torch.matmul(gram_diag.unsqueeze(-1), torch.ones(1, L).to(coordinates.device))
    # diag_1: L x L
    diag_2 = torch.transpose(diag_1, dim0=0, dim1=1)
    # diag_2: L x L
    squared_distance_matrix = diag_1 + diag_2 - (2 * gram_matrix)
    distance_matrix = torch.zeros_like(squared_distance_matrix)
    mask = ~torch.eye(L, dtype=torch.bool).to(coordinates.device)
    distance_matrix[mask] = torch.sqrt( squared_distance_matrix.masked_select(mask) )
    return distance_matrix

# In reality, pred_coordinates is an output of the network, but we initialize it here for a minimal working example
L = 10
pred_coordinates = torch.randn(L, 3, requires_grad=True)
true_coordinates = torch.randn(L, 3, requires_grad=False)
obj = torch.nn.MSELoss()
optimizer = torch.optim.Adam([pred_coordinates])

for i in range(500):
    pred_distance_matrix = compute_distance_matrix(pred_coordinates)
    true_distance_matrix = compute_distance_matrix(true_coordinates)
    loss = obj(pred_distance_matrix, true_distance_matrix)
    loss.backward()
    print(loss.item())
    optimizer.step()

which gives:
1.222102403640747
1.2191187143325806
1.2162436246871948
1.2133947610855103
1.210543155670166
1.2076761722564697
1.204787015914917
1.2018715143203735
1.198927402496338
1.1959534883499146
1.1929489374160767
1.1899129152297974
1.1868458986282349
1.1837480068206787
1.180619239807129
1.1774601936340332
1.174271583557129
...


Instruction: Why does my Pytorch tensor size change and contain NaNs after some batches?

Input:
I am Training a Pytorch model. After some time, even if on shuffle, the model contains, besides a few finite tensorrows only NaN values:
tensor([[[    nan,     nan,     nan,  ...,     nan,     nan,     nan],
         [    nan,     nan,     nan,  ...,     nan,     nan,     nan],
         [    nan,     nan,     nan,  ...,     nan,     nan,     nan],
         ...,
         [ 1.4641,  0.0360, -1.1528,  ..., -2.3592, -2.6310,  6.3893],
         [    nan,     nan,     nan,  ...,     nan,     nan,     nan],
         [    nan,     nan,     nan,  ...,     nan,     nan,     nan]]],
       device='cuda:0', grad_fn=<AddBackward0>)

The detect_anomaly functions return:
  File "TestDownload.py", line 701, in <module>
    main(learning_rate, batch_size, epochs, experiment)
  File "TestDownload.py", line 635, in main
    train(model, device, train_loader, criterion, optimizer, scheduler, epoch, iter_meter, experiment)
  File "TestDownload.py", line 486, in train
    output = F.log_softmax(output, dim=2)
  File "\lib\site-packages\torch\nn\functional.py", line 1672, in log_softmax
    ret = input.log_softmax(dim)
 (function _print_stack) Traceback (most recent call last):
  File "TestDownload.py", line 701, in <module>
    main(learning_rate, batch_size, epochs, experiment)
  File "TestDownload.py", line 635, in main
    train(model, device, train_loader, criterion, optimizer, scheduler, epoch, iter_meter, experiment)
  File "TestDownload.py", line 490, in train
    loss.backward()
  File "\lib\site-packages\comet_ml\monkey_patching.py", line 317, in wrapper
    return_value = original(*args, **kwargs)
  File "\lib\site-packages\torch\tensor.py", line 245, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "\lib\site-packages\torch\autograd\__init__.py", line 145, in backward
    Variable._execution_engine.run_backward(
RuntimeError: Function 'LogSoftmaxBackward' returned nan values in its 0th output.

in reference to the next line output = F.log_softmax(output, dim=2)
It shows another error if I just do it with try-except: (when the loss function is running on a tensor containing NaNs)
[W ..\torch\csrc\autograd\python_anomaly_mode.cpp:104] Warning: Error detected in CtcLossBackward. Traceback of forward call that caused the error:
  File "TestDownload.py", line 734, in <module>
    # In[ ]:
  File "TestDownload.py", line 667, in main
    test(model, device, test_loader, criterion, epoch, iter_meter, experiment)
  File "TestDownload.py", line 517, in train
    loss.backward()
  File "\lib\site-packages\torch\nn\modules\module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "\lib\site-packages\torch\nn\modules\loss.py", line 1590, in forward
    return F.ctc_loss(log_probs, targets, input_lengths, target_lengths, self.blank, self.reduction,
  File "\lib\site-packages\torch\nn\functional.py", line 2307, in ctc_loss
    return torch.ctc_loss(
 (function _print_stack)
Traceback (most recent call last):
  File "TestDownload.py", line 518, in train
  File "\lib\site-packages\comet_ml\monkey_patching.py", line 317, in wrapper
    return_value = original(*args, **kwargs)
  File "\lib\site-packages\torch\tensor.py", line 245, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
  File "\lib\site-packages\torch\autograd\__init__.py", line 145, in backward
    Variable._execution_engine.run_backward(
RuntimeError: Function 'CtcLossBackward' returned nan values in its 0th output.

A normal tensor should look like this:
tensor([[[-3.3904, -3.4340, -3.3703,  ..., -3.3613, -3.5098, -3.4344]],

        [[-3.3760, -3.2948, -3.2673,  ..., -3.4039, -3.3827, -3.3919]],

        [[-3.3857, -3.3358, -3.3901,  ..., -3.4686, -3.4749, -3.3826]],

        ...,

        [[-3.3568, -3.3502, -3.4416,  ..., -3.4463, -3.4921, -3.3769]],

        [[-3.4379, -3.3508, -3.3610,  ..., -3.3707, -3.4030, -3.4244]],

        [[-3.3919, -3.4513, -3.3565,  ..., -3.2714, -3.3984, -3.3643]]],
       device='cuda:0', grad_fn=<TransposeBackward0>)

Please notice the double brackets, if they are import.
Code:
for batch_idx, _data in enumerate(train_loader):
    spectrograms, labels, input_lengths, label_lengths = _data
    spectrograms, labels = spectrograms.to(device), labels.to(device)
    optimizer.zero_grad()

    output = model(spectrograms)
    output = F.log_softmax(output, dim=2)
    output = output.transpose(0, 1)  # (time, batch, n_class) # X, 1, 29
    loss = criterion(output, labels, input_lengths, label_lengths)
    loss.backward()
    optimizer.step()
    scheduler.step()
    iter_meter.step()

Additionally, I tried to run it with a bigger batch size (current batch size:1, bigger batch size: 6) and it run without errors until 40% of the first epoch in which I got this error.

Cuda run out of memory

Also, I tried to normalize the data torchaudio.transforms.MelSpectrogram(sample_rate=16000, n_mels=128, normalized=True)
And reducing the learning rate from 5e-4 to 5e-5 did not help either.
Additional information: My dataset contains nearly 300000 .wav files and the error came at 3-10% runtime in the first epoch.
I appreciate any hints and I will gladly submit further information.

Output:
The source of error can be a corrupted input or label, which would contain a NaN of inf value.
You can check that there is no NaN value in a tensor with
torch.isnan(tensor).any()

Or that all values in a tensor are neither inf nor NaN with
torch.isfinite(tensor).all()


Instruction: PyTorch DDP: Finding the cause of "Expected to mark a variable ready only once"

Input:
I'm extending a complex model (already with DistributedDataParallel with find_unused_parameters set to True) in PyTorch on detectron2.
I've added a new layer generating some additional output to the original network - initially, that layer was frozen (requires_grad = False) and everything was working fine.
I later decided to unfreeze this layer.
Unfortunately, this results in this error on multiple GPUs:
-- Process 0 terminated with the following error:
Traceback (most recent call last):
...
  File "/xxx/trainer.py", line 585, in some_method:
    losses.backward()
  File "/xxx/python3.7/site-packages/torch/tensor.py", line 221, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/xxx/python3.7/site-packages/torch/autograd/__init__.py", line 132, in backward
    allow_unreachable=True)  # allow_unreachable flag

RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons:
1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes
2) Reused parameters in multiple reentrant backward passes. For example, if you use  multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases yet.

Clearly, the error is related to that unfrozen layer, since everything was working before unfreezing it.
This means that I need to check every change I made.
I wonder if there's a way/trick to determine which particular tensor/operation causes this behaviour? In other words - how can I speed up the debugging process?

Output:
With the help of the PyTorch community, I moved forward (see the original discussion here).
I updated my PyTorch to 1.9.0 (was using 1.7.0 before). Now I the error is a bit more informative:
RuntimeError: Expected to mark a variable ready only once. This error is caused by one of the following reasons: 1) Use of a module parameter outside the `forward` function. Please make sure model parameters are not shared across multiple concurrent forward-backward passes. or try to use _set_static_graph() as a workaround if this module graph does not change during training loop.2) Reused parameters in multiple reentrant backward passes. For example, if you use multiple `checkpoint` functions to wrap the same part of your model, it would result in the same set of parameters been used by different reentrant backward passes multiple times, and hence marking a variable ready multiple times. DDP does not support such use cases in default. You can try to use _set_static_graph() as a workaround if your module graph does not change over iterations.
Parameter at index 73 has been marked as ready twice. This means that multiple autograd engine  hooks have fired for this particular parameter during this iteration. You can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print parameter names for further debugging.

After setting the environmental variable TORCH_DISTRIBUTED_DEBUG to DETAIL (this requires PyTorch 1.9.0!) I got the name of the problematic variable:
Parameter at index 73 with name roi_heads.box_predictor.xxx.bias has been marked as ready twice. This means that multiple autograd engine  hooks have fired for this particular parameter during this iteration.

So, technically, there was a problem with roi_heads.box_predictor.xxx.
Could not find one.
However, with version 1.9.0 the console also outputs this:
[W reducer.cpp:1158] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())

Turned out to be the root cause of the problem.
Switching find_unused_parameters to False in the DDP constructor makes the training run normally.
Not sure why this was the cause, but I don’t mind, since the network is trained now.
Anyhow, I hope this will help someone who struggles with similar debugging problems.


Instruction: Vectorizing ARD (Automatic Relevance Determination) kernel implementation in Gaussian processes

Input:
I am trying to implement an ARD kernel with NumPy as given in the GPML book (M3 from Equation 5.2).

I am struggling in vectorizing this equation for NxM kernel computation. I have tried the following non-vectorized version. Can someone help in vectorizing this in NumPy/PyTorch?
import numpy as np
N = 30 # Number of data points in X1
M = 40 # Number of data points in X2
D = 6  # Number of features (ARD dimensions)

X1 = np.random.rand(N, D)
X2 = np.random.rand(M, D)
Lambda = np.random.rand(D, 1)
L_inv = np.diag(np.random.rand(D))
sigma_f = np.random.rand()

K = np.empty((N, M))
for n in range(N):
  for m in range(M):
    M3 = Lambda@Lambda.T + L_inv**2
    d = (X1[n,:] - X2[m,:]).reshape(-1,1)
    K[n, m] = sigma_f**2 * np.exp(-0.5 * d.T@M3@d)

Output:
We can use the rules of broadcasting and the neat NumPy function einsum to vectorize array operations. In few words, broadcasting allows us to operate with arrays in one-liners by adding new dimensions to the resulting array, while einsum allows us to perform operations with multiple arrays by explicitly working in the index notation (instead of matrices).
Luckily, no loops are necessary to calculate your kernel. Please see below the vectorized solution, ARD_kernel function, which is about 30x faster in my machine than the original loopy version. Now, einsum is usually as fast as it gets, but it's possible that there are faster methods though, I've not checked anything else (e.g. usual @ operator instead of einsum).
Also, there is a missing term in the code (the Kronecker delta), I don't know if it was omitted in purpose (let me know if you have problems implementing it and I'll edit the answer).
import numpy as np
N = 300 # Number of data points in X1
M = 400 # Number of data points in X2
D = 6  # Number of features (ARD dimensions)

np.random.seed(1)  # Fix random seed for reproducibility
X1 = np.random.rand(N, D)
X2 = np.random.rand(M, D)
Lambda = np.random.rand(D, 1)
L_inv = np.diag(np.random.rand(D))
sigma_f = np.random.rand()

# Loopy function
def ARD_kernel_loops(X1, X2, Lambda, L_inv, sigma_f):
    K = np.empty((N, M))
    M3 = Lambda@Lambda.T + L_inv**2
    for n in range(N):
        for m in range(M):
            d = (X1[n,:] - X2[m,:]).reshape(-1,1)
            K[n, m] = np.exp(-0.5 * d.T@M3@d)
    return K * sigma_f**2

# Vectorized function
def ARD_kernel(X1, X2, Lambda, L_inv, sigma_f):
    M3 = Lambda.squeeze()*Lambda + L_inv**2  # Use broadcasting to avoid transpose
    d = X1[:,None] - X2[None,...]            #  Use broadcasting to avoid loops

    # order=F for memory layout (as your arrays are (N,M,D) instead of (D,N,M))
    return sigma_f**2 * np.exp(-0.5 * np.einsum("ijk,kl,ijl->ij", d, M3, d, order = 'F'))


Instruction: Converted model from keras h5 to pytorch - fully connected layer mismatch

Input:
I have converted two models (vgg16 and resnet50) from Keras with TensorFlow backend (from as model.save file) into PyTorch using mmdnn. This was done with the following:
mmconvert -sf keras -iw vgg.h5 -df pytorch -om keras_to_torch.pt

A = imp.load_source('MainModel','/weights/keras_to_torch.py')
model = torch.load('/weights/keras_to_torch.pt')

Predicting on the same data set gave me a different set of results so I investigated further.
I can see that the weights for all the convolutional layers are the same (after transposing), however the weights of the fully connected layers at the end are not.
Is there a reason this should be? As i understand they should be equivalent

Output:
The problem must be in the way you defined your keras model, since I cannot replicate the issue using the h5 file that is provided using the MMdnn package. If you want to use the resnet50 and VGG19 model you can get the correct weights as follows:

start MMdnn container as specified in the documentation
download keras model for resnet50

mmdownload -f keras -n resnet50 -o ./

convert to pytorch model

mmconvert -sf keras -iw ./imagenet_resnet50.h5 -df pytorch -om keras_to_torch.pt

Then extract the produced numpy file, keras_to_torch.pt and  keras_to_torch.py  from the docker container (and  imagenet_resnet50.h5  for comparison).
In Python load the keras model with
import keras
model = load_model('imagenet_resnet50.h5')

and the torch model using
import imp
import torch
torch_weights = # path_to_the_numpy_weights
A = imp.load_source('MainModel','keras_to_torch.py')
weights_torch = A.load_weights(torch_weights)
model_torch = A.KitModel(torch_weights)

I also had to set allow_pickle = True in the load_weights(weight_file) function at the beginning of the keras_to_torch.py file. The torch.load('/weights/keras_to_torch.pt') variant threw an error for me unfortunately.
Print the weights of the last densely connected layer
# keras model
model.layers[-1].weights

# Output:
#tensor([[-0.0149,  0.0113, -0.0507,  ..., -0.0218, -0.0776,  0.0102],
#        [-0.0029,  0.0032,  0.0195,  ...,  0.0362,  0.0035, -0.0332],
#        [-0.0175,  0.0081,  0.0085,  ..., -0.0302,  0.0549, -0.0251],
#        ...,
#        [ 0.0253,  0.0630,  0.0204,  ..., -0.0051, -0.0354, -0.0131],
#        [-0.0062, -0.0162, -0.0122,  ...,  0.0138,  0.0409, -0.0186],
#        [-0.0267,  0.0131, -0.0185,  ...,  0.0630,  0.0256, -0.0069]])

# torch model (make sure to transpose)
model_torch.fc1000.weight.data.T

# Output:
#[<tf.Variable 'fc1000/kernel:0' shape=(2048, 1000) dtype=float32, numpy=
# array([[-0.01490746,  0.0113374 , -0.05073728, ..., -0.02179668,
#         -0.07764222,  0.01018347],
#        [-0.00294467,  0.00319835,  0.01953556, ...,  0.03623696,
#          0.00350259, -0.03321117],
#        [-0.01751374,  0.00807406,  0.00851311, ..., -0.03024036,
#          0.05494978, -0.02511911],
#        ...,
#        [ 0.025289  ,  0.0630148 ,  0.02041481, ..., -0.00508354,
#         -0.03542514, -0.01306196],
#        [-0.00623157, -0.01624131, -0.01221174, ...,  0.01376359,
#          0.04087579, -0.0185826 ],
#        [-0.02668471,  0.0130982 , -0.01847764, ...,  0.06304929
#...

The weights of the keras and torch model coincide as desired (up to 4 digits or so).
This solution works as long as you don't want to update the VGG and ResNet weights in keras before converting them to Pytorch.
If you do need to update the model weights before converting you should share your code for creating the Keras model. You could further inspect how the imagenet_resnet50.h5 obtained with mmdownload model differs from the one you saved with model.save in keras and correct for any differences.


Instruction: How to test custom Faster RCNN model(using Detectron 2 and pytorch) on video?

Input:
I have trained a Faster RCNN model on a custom dataset for object detection and want to test it on Videos. I could test the results on images but am stuck on how to do that for a video.
Here is the code for inference on images:
cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
cfg.DATASETS.TEST = ("my_dataset_test", )
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set the testing threshold for this model
predictor = DefaultPredictor(cfg)
test_metadata = MetadataCatalog.get("my_dataset_test")

from detectron2.utils.visualizer import ColorMode
import glob

for imageName in glob.glob('/content/test/*jpg'):
  im = cv2.imread(imageName)
  outputs = predictor(im)
  v = Visualizer(im[:, :, ::-1],
                metadata=test_metadata,
                scale=0.8
                 )
  out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
  cv2_imshow(out.get_image()[:, :, ::-1])

Please somebody let me know how tweak this code to work for detection on videos?
Platform used: Google Colab
Tech Stack:Detectron 2, Pytorch

Output:
Check this loop out:
from detectron2.utils.visualizer import ColorMode
import glob
import cv2
from google.colab.patches import cv2_imshow

cap = cv2.VideoCapture('/path/to/video')
while cap.isOpened():
    ret, frame = cap.read()
    # if frame is read correctly ret is True
    if not ret:
        break

    outputs = predictor(im)
    v = Visualizer(im[:, :, ::-1],
    metadata=test_metadata,
    scale=0.8
    )
    out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
    cv2_imshow(out.get_image()[:, :, ::-1])

    if cv2.waitKey(1) == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


Instruction: TypeError: forward() takes 2 positional arguments but 4 were given, Pytorch

Input:
I am trying to write a GAN generator based on Densenet and Deconv method. I am new to PyTorch and unable to figure out
TypeError: forward() takes 2 positional arguments but 4 were given.

I tried the approach as suggested in
Pytorch TypeError: forward() takes 2 positional arguments but 4 were given
but I cannot figure out the solution.
My code:
class DenseLayer(nn.Module):
  def __init__(self, in_size, out_size, drop_rate=0.0):
    super(DenseLayer, self).__init__()
    self.bottleneck = nn.Sequential() # define bottleneck layers
    self.bottleneck.add_module('btch1', nn.BatchNorm2d(in_size))
    self.bottleneck.add_module('relu1', nn.ReLU(inplace=True))
    self.bottleneck.add_module('conv1', nn.ConvTranspose2d(in_size, int(out_size/4), kernel_size=1, stride=1, padding=0, bias=False))

    self.basic = nn.Sequential() # define basic block
    self.basic.add_module('btch2', nn.BatchNorm2d(int(out_size/4)))
    self.basic.add_module('relu2', nn.ReLU(inplace=True))
    self.basic.add_module('conv2', nn.ConvTranspose2d(int(out_size/4), out_size, kernel_size=3, stride=1, padding=1, bias=False))

    self.droprate = drop_rate

  def forward(self, input):
    out = self.bottleneck(input)
    if self.droprate > 0:
      out = F.dropout(out, p=self.droprate, inplace=False, training=self.training)

    out = self.basic(out)
    if self.droprate > 0:
      out = F.dropout(out, p=self.droprate, inplace=False, training=self.training)
    return torch.cat((x,out), 1)

class DenseBlock(nn.Module):
  def __init__(self, num_layers, in_size, growth_rate, block, droprate=0.0):
    super(DenseBlock, self).__init__()
    self.layer = self._make_layer(block, in_size, growth_rate, num_layers, droprate)

  def _make_layer(self, block, in_size, growth_rate, num_layers, droprate):
    layers = []
    for i in range(num_layers):
      layers.append(block(in_size, in_size-i*growth_rate, droprate))
    return nn.Sequential(*layers)

  def forward(self, input):
    return self.layer(input)

class MGenDenseNet(nn.Module):
  def __init__(self, ngpu, growth_rate=32, block_config=(16,24,12,6), in_size=1024, drop_rate=0.0):
    super(MGenDenseNet, self).__init__()
    self.ngpu = ngpu
    self.features = nn.Sequential()
    self.features.add_module('btch0', nn.BatchNorm2d(in_size))

    block = DenseLayer
    num_features = in_size
    for i, num_layers in enumerate(block_config):
      block = DenseBlock(num_layers=num_layers, in_size=num_features, growth_rate=growth_rate, block=block, droprate=drop_rate) ### Error thrown on this line
      self.features.add_module('denseblock{}'.format(i+1), block)
      num_features -= num_layers*growth_rate

      if i!=len(block_config)-1:
        trans = TransitionLayer(in_size=num_features, out_size=num_features*2, drop_rate=drop_rate)
        self.features.add_module('transitionblock{}'.format(i+1), trans)
        num_features *= 2

    self.features.add_module('convfinal', nn.ConvTranspose2d(num_features, 3, kernel_size=7, stride=2, padding=3, bias=False))
    self.features.add_module('Tanh', nn.Tanh())

  def forward(self, input):
    return self.features(input)

mGen = MGenDenseNet(ngpu).to(device)
mGen.apply(weights_init)

print(mGen)

Output:
class MGenDenseNet(nn.Module):
  def __init__(self, ngpu, growth_rate=32, block_config=(16,24,12,6), in_size=1024, drop_rate=0.0):
    super(MGenDenseNet, self).__init__()
    import pdb; pdb.set_trace()
    self.ngpu = ngpu
    self.features = nn.Sequential()
    self.features.add_module('btch0', nn.BatchNorm2d(in_size))
    block_placeholder = DenseLayer <<<<
    num_features = in_size
    for i, num_layers in enumerate(block_config):
      block = DenseBlock(num_layers=num_layers, in_size=num_features, growth_rate=growth_rate, block=block_placeholder, droprate=drop_rate) <<<< look at change
      self.features.add_module('denseblock{}'.format(i+1), block)
      num_features -= num_layers*growth_rate
    self.features.add_module('convfinal', nn.ConvTranspose2d(num_features, 3, kernel_size=7, stride=2, padding=3, bias=False))
    self.features.add_module('Tanh', nn.Tanh())
  def forward(self, input):
    return self.features(input)

It is because you define block as DenseLayer, then reassign block it to an initalized DenseBlock() and then pass that as block=block. So after one iteration through the for loop it is passing a DenseBlock() object instead of DenseLayer so it's wrongly using the forward pass.
Just change block = DenseLayer to block_placeholder and use that variable instead.
I spotted this by placing a debugger in your code and noticing that the DenseBlock line only fails on second call.


Instruction: How to solve gradient-exploding in YOLO v1

Input:
Now I am trying to train object detection - YOLOv1 using this code. At the beginning I was using momentum and weight_decay but the training loss after couples of epochs becomes NaN. As far as I know it's because of gradient exploding, so I have searched some ways to get rid of this NaN then I ignored momentum and weight decay. As a result I did not get any NaN, however my model could not converge as I expected. When I calculated mAP it was only 0.29. I am using VOC 2007 and 2012 data for training and as a test set VOC 2007 test.
So my questions are followings:

How can I get rid of NaN while training?
Where can I get best training configurations?
Is gradient exploding is normal in Object Detection task?
Is it normal in YOLOv1 getting 1.1Gb weights after training?

Would appreciate any suggestions here.

Output:
After checking your code, I saw after the first epoch, you would set the learning rate to 0.01 until epoch 75. In my opinion, that large learning rate is the main reason made your parameters became vanishing/exploding. Normally, the learning rate is scaling around 0.001 with the factor of 2,1,0.1.
Follow the config in this repo (the most famous repo implement YOLOv1 according to paperwithcode), you can see their configurations setup. You can follow their hyper-parameters, momentum=0.9 and decay=0.0005 in your question.
Note: please be careful that the batch norm momentum in Tensorflow = 1 - momentum in Pytorch.
Finally, your number of parameters before and after training should be the same, so if your model is heavier/lighter after the training process, it means there is something wrong with your training code.


Instruction: Extending Pytorch: Python vs. C++ vs. CUDA

Input:
I have been trying to implement a custom Conv2d module where grad_input (dx) and grad_weight (dw) are calculated by using different grad_output (dy) values. I implemented this by extending torch.autograd as in Pytorch tutorials.
However I am confused by the information in this link.

Is extending the autograd.Function not enough?
What is the difference
between writing a new autograd function in Python vs C++?
How about
the CUDA implementations in
/torch/nn/blob/master/lib/THNN/generic/SpatialConvolutionMM.c where
dx and dw calculated? Should I change them too?

Here is my custom function:
class myCustomConv2d(torch.autograd.Function):
@staticmethod
def forward(ctx, x, w, bias=None, stride=1, padding=0, dilation=1, groups=1):
    ctx.save_for_backward(x, w, bias)
    ctx.stride = stride
    ctx.padding = padding
    ctx.dilation = dilation
    ctx.groups = groups
    out = F.conv2d(x, w, bias, stride, padding, dilation, groups)
    return out

@staticmethod
def backward(ctx, grad_output):
    input, weight, bias = ctx.saved_tensors
    stride = ctx.stride
    padding = ctx.padding
    dilation = ctx.dilation
    groups = ctx.groups
    grad_input = grad_weight = grad_bias = None

    dy_for_inputs = myspecialfunction1(grad_output)
    dy_for_weights = myspecialfunction2(grad_output)

    grad_input = torch.nn.grad.conv2d_input(input.shape, weight, dy_for_inputs , stride, padding, dilation, groups)
    grad_weight = torch.nn.grad.conv2d_weight(input, weight.shape, dy_for_weights , stride, padding, dilation, groups)

    if bias is not None and ctx.needs_input_grad[2]:
        grad_bias = dy_for_weights .sum((0,2,3)).squeeze(0)

    return grad_input, grad_weight, grad_bias, None, None, None, None

Output:
Is extending the autograd.Function not enough?

It is enough if your code reuses Pytorch components wrapped within Python interface (which seems to be the case). Gradient is composed automatically.

What is the difference between writing a new autograd function in Python vs C++?

Performance, the more custom your operation is (and the harder it is to compose it from exstsing Pytorch operations), the more performance improvement you would obtain.

How about the CUDA implementations in /torch/nn/blob/master/lib/THNN/generic/SpatialConvolutionMM.c where dx and dw calculated? Should I change them too?

No need for that, unless you want to create specialized ops for CUDA


Instruction: Pairwise difference between tensors with PyTorch

Input:
I have the following problem: given two tensors X of shape (T,n,d) and Y of shape (R,m,d), I want to compute the tensor D of shape (T,R,n,m,d) such that, for all 0 <= t < T, 1 <= r < R, 1 <= i < n, 1 <= j < m,
D[t,r,i,j] = X[t,i] - Y[r,j]
Can we compute D with Pytorch without using any loop?
I know that when given two tensors X of shape (n,d) and Y of shape (m,d) we can compute the tensor D of shape (n,m,d) such that, for all 1 <= i < n, 1 <= j < m, D[i,j] = X[i] - Y[j] using
x.unsqueeze(1) - y
What I am looking for is a similar trick for the initial problem.

Output:
I think the following snippet might achieve what you need :
# reshapes X to (T, R, n, m, d)
X_rs = X.view(T, 1, n, 1, d).expand(-1, R, -1, m, -1)
# reshapes Y to (T, R, n, m, d)
Y_rs = Y.view(1, R, 1, m, d).expand(T, -1, n, -1, -1)
# Compute the coefficient-wise difference
D = X_rs - Y_rs

Note that the expand operation does not allocate new memory. It's just the generalization of the "trick" you mentioned


Instruction: CNN model is not learn well

Input:
I started to learn CNN implementation in PyTorch, and I tried to build CNNs to process the grayscale images with 4 classes from 0 to 3. I got in the beginning accuracy around 0.55. The maximum accuracy I got is ~ 0.683%.
I tried SGD and Adam optimizer with different values for lr and batch_size, but the accuracy is still low.
I used data Augmentation to create more samples, around 4k.
I cannot improve accuracy further and wondered if I could get some advices about what I need to change in CNN structure to increase accuracy.
Loss starts around: Loss: [1.497] then decreases near: Loss: [0.001] then fluctuated up and down around this value.
I spent time reading about similar problems but without luck.
I am using nn.CrossEntropyLoss() for my loss_fn. I don't use softmax for dense layer.
This is the Summary of the CNN model:
-------------------------------------------------------------
        Layer (type)               Output Shape         Param #
=============================================================
            Conv2d-1         [-1, 32, 128, 128]             320
              ReLU-2         [-1, 32, 128, 128]               0
       BatchNorm2d-3         [-1, 32, 128, 128]              64
         MaxPool2d-4           [-1, 32, 64, 64]               0
            Conv2d-5           [-1, 64, 64, 64]          18,496
              ReLU-6           [-1, 64, 64, 64]               0
       BatchNorm2d-7           [-1, 64, 64, 64]             128
         MaxPool2d-8           [-1, 64, 32, 32]               0
            Conv2d-9          [-1, 128, 32, 32]          73,856
             ReLU-10          [-1, 128, 32, 32]               0
      BatchNorm2d-11          [-1, 128, 32, 32]             256
        MaxPool2d-12          [-1, 128, 16, 16]               0
          Flatten-13                [-1, 32768]               0
           Linear-14                  [-1, 512]      16,777,728
             ReLU-15                  [-1, 512]               0
          Dropout-16                  [-1, 512]               0
           Linear-17                    [-1, 4]           2,052
============================================================

I would appreciate the help.

Output:
How many images are in the train set ? the test set ? What are the size of the images ? How would you consider the difficulty of classification of the images ? Do you think it should be simple or difficult ?
According to the numbers you have, you're overfitting as your loss is near 0 (meaning nothing much will retropropagate to the weights, i.e your model won't change anymore) and your 68.3% (it's a typo right ?) is from the test set (I suppose). So you don't have any problem to train the network which is a good point.
Then you can search ways of countering overfitting online and here is some "classical" possibilities :
- you may raise the dropout parameter
- putting some regularizer (L1 or L2) to constraint the learning
- early stopping using a validation set
- using a classical and/or lighter convolutional network (resnet,inception) with/without pretrained weights. This latter also depends on your images type (natural, biomedical ...)
- ... a lot more or less difficult to implement
Also technically you are using a softmax layer as it's included in the crossentropyloss of pytorch.


Instruction: Creating random linearly independent vectors

Input:
I can generate a random vector with
a = torch.rand(10)
>>> tensor([0.6432, 0.7512, 0.8835, 0.4268, 0.7681, 0.4709, 0.2722, 0.0510, 0.8463,
        0.9003])

However, how can I generate N more vectors (e.g. N=5: b,c,d,e,f) that will be linearly independent of the other vectors (i.e a is independent of any of b,c,d,e,f, b is independent of any of a,c,d,e,f, etc.)?

Output:
I can think of two ways to achieve this. However, there are some restrictions to each of them.
For an n-dimensional space, the first way is to create n random vectors and check for linear independence using determinants. Check out here for more information about the process.
The second way is to generate vectors one by one. Create a random vector, then using this solution, create a new independent vector, then create the third one, and so on.


Instruction: Elegant way to get a symmetric Torch Tensor over diagonal

Input:
Is there an elegant way to build a Torch.Tensor like this from a given set of values?
Here is an 3x3 example, but in my application I would have a matrix of any odd-size.
A function call gen_matrix([a, b, c, d, e, f]) should generate

Output:
You can use torch.triu_indices() to achieve this. Note that for a general N x N symmetric matrix, there can be atmost N(N+1)/2 unique elements which are distributed over the matrix.
The vals tensor here stores the elements you want to build the symmetric matrix with. You can simply use your set of values in place of vals. Only constraint is, it should have N(N+1)/2 elements in it.
Short answer:
N = 5 # your choice goes here
vals = torch.arange(N*(N+1)/2) + 1 # values

A = torch.zeros(N, N)
i, j = torch.triu_indices(N, N)
A[i, j] = vals
A.T[i, j] = vals

Example:
>>> N = 5
>>> vals = torch.arange(N*(N+1)/2) + 1

>>> A = torch.zeros(N, N)
>>> i, j = torch.triu_indices(N, N)
>>> A[i, j] = vals
>>> A.T[i, j] = vals

>>> vals
 tensor([ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10., 11., 12., 13., 14.,
         15.])
>>> A
 tensor([[ 1.,  2.,  3.,  4.,  5.],
         [ 2.,  6.,  7.,  8.,  9.],
         [ 3.,  7., 10., 11., 12.],
         [ 4.,  8., 11., 13., 14.],
         [ 5.,  9., 12., 14., 15.]])


Instruction: How to convert Bounding Box coordinates to Yolo Coordinates with Python?

Input:
I need to convert the coordinates. I have this format <left> <top> <width> <height>
And I need x_center y_center width height in this format. How can I do it?

Output:
The center is just the middle of your bounding box. So just add half of the bounding box width or height to yout top-left coordinate. Width and height remain unchanged
x_center = left + width / 2
y_center = top + height / 2


Instruction: PyTorch: how to select values from a multidimensional tensor using a multidimensional index?

Input:
Given a tensor input of shape (5, 2, 2):
tensor([[[ 0,  1],
         [ 2,  3]],

        [[ 4,  5],
         [ 6,  7]],

        [[ 8,  9],
         [10, 11]],

        [[12, 13],
         [14, 15]],

        [[16, 17],
         [18, 19]]])

and a tensor index of shape (2,2):
tensor([[4, 3],
        [4, 2]])

How do I obtain the following output:
tensor([[16, 13],
        [18, 11]])

To put it into context: I have 5 input images of size 2x2 pixels (stored in input). Now I want to combine these 5 images into a single image of size 2x2 pixels, where index determines for each pixel from which input image it should be copied.
Example: starting at the top-left index[0,0] == 4, I take pixel value input[4,0,0] == 16. Then continue to index[0,1] == 3, I take pixel value input[3,0,1] == 13 and so on.

Output:
The solution I found is almost the same as this one:
indices = np.indices(input.shape)
indices[0] = index
input[tuple(indices)][0]

which gives the desired output
tensor([[16, 13],
        [18, 11]])


Instruction: PyTorch: .movedim() vs. .moveaxis() vs. .permute()

Input:
I'm completely new to PyTorch, and I was wondering if there's anything I'm missing when it comes to the .moveaxis() and .movedim() methods. The outputs are the exact same for the same arguments. Also can't both of these methods be replaced by .permute()?

An example for reference:
import torch

mytensor = torch.randn(3,6,3,1,7,21,4)

t_md = torch.movedim(mytensor, 2, 5)
t_ma = torch.moveaxis(mytensor, 2, 5)

print(t_md.shape, t_ma.shape)
print(torch.allclose(t_md, t_ma))

t_p = torch.permute(mytensor, (0, 1, 3, 4, 5, 2, 6))

print(t_p.shape)
print(torch.allclose(t_md, t_p))

Output:
Yes, moveaxis is an alias of movedim (analogous to swapaxes and swapdims).1
Yes, this functionality can be achieved with permute, but moving one axis while keeping the relative positions of all others is a common enough use-case to warrant its own syntactic sugar.

The terminology is taken from numpy:

Alias for torch.movedim().
This function is equivalent to NumPy’s moveaxis function.


Instruction: PyTorch Sampler: if specified, the next iteration samples the same subset or another subset?

Input:
If I have the code like:
dataset = Dataset(...)
sampler = RandomSampler(...)
dataloader = DataLoader(..., sampler=sampler)

Then whenever I call:
for data, label in dataloader:
   ...

The returned tuple data, label is the same subset or different subset compared to the last call?

Output:
It is different subset compared to the last call. I modify example here for your question:
data = torch.rand(10,1)
dataset = torch.utils.data.TensorDataset(torch.arange(len(data)),data)
index,_ = dataset[:]

sampler = torch.utils.data.RandomSampler(index)
loader = torch.utils.data.DataLoader(dataset, sampler=sampler, batch_size=3)

for i in range(2):
  for data, label in loader:
    print(data, label)
  print("------------")


Instruction: Accuracy value goes up and down on the training process

Input:
After training the network I noticed that accuracy goes up and down. Initially I thought it is caused by the learning rate, but it is set to quite small value. Please check the screenshot attached.
Plot Accuracy Screenshot
My network (in Pytorch) looks as follow:
class Network(nn.Module):
    def __init__(self):
        super(Network,self).__init__()

    self.layer1 = nn.Sequential(
        nn.Conv2d(3,16,kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2)
    )

    self.layer2 = nn.Sequential(
        nn.Conv2d(16,32, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2)
        )

    self.layer3 = nn.Sequential(
        nn.Conv2d(32,64, kernel_size=3),
        nn.ReLU(),
        nn.MaxPool2d(2)
    )

    self.fc1 = nn.Linear(17*17*64,512)
    self.fc2 = nn.Linear(512,1)
    self.relu = nn.ReLU()
    self.sigmoid = nn.Sigmoid()

def forward(self,x):
    out = self.layer1(x)
    out = self.layer2(out)
    out = self.layer3(out)
    out = out.view(out.size(0),-1)
    out = self.relu(self.fc1(out))
    out = self.fc2(out)
    out = torch.sigmoid(out)
    return out

I am using RMSprop as optimizer and BCELoss as criterion. The learning rate is set to 0.001
Here is the training process:
epochs = 15
itr = 1
p_itr = 100
model.train()
total_loss = 0
loss_list = []
acc_list = []
for epoch in range(epochs):
    for samples, labels in train_loader:
        samples, labels = samples.to(device), labels.to(device)
        optimizer.zero_grad()
        output = model(samples)
        labels = labels.unsqueeze(-1)
        labels = labels.float()
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        total_loss += loss.item()
        scheduler.step()

    if itr%p_itr == 0:
        pred = torch.argmax(output, dim=1)
        correct = pred.eq(labels)
        acc = torch.mean(correct.float())
        print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, Accuracy: {:.3f}'.format(epoch+1, epochs, itr, total_loss/p_itr, acc))
        loss_list.append(total_loss/p_itr)
        acc_list.append(acc)
        total_loss = 0

    itr += 1

My dataset is quite small - 2000 train and 1000 validation (binary classification 0/1). I wanted to do the 80/20 split but I was asked to keep it like that. I was thinking that the architecture might be too complex for such a small dataset.
Any hits what may cause such jumps in the training process?

Output:
Your code here is wrong: pred = torch.argmax(output, dim=1)
This line using for multiclass classification with Cross-Entropy Loss.
Your task is binary classification so the pred values are wrong. Change to:
if itr%p_itr == 0:
  pred = torch.round(output)
  ....

You can change your optimizer to Adam, SGD, or RMSprop to find the suitable optimizer that helps your model coverage faster.
Also change the forward() function:
def forward(self,x):
   out = self.layer1(x)
   out = self.layer2(out)
   out = self.layer3(out)
   out = out.view(out.size(0),-1)
   out = self.relu(self.fc1(out))
   out = self.fc2(out)
   return self.sigmoid(out) #use your forward is ok, but this cleaner


Instruction: RunTimeError during one hot encoding

Input:
I have a dataset where class values go from -2 to 2 by 1 step (i.e., -2,-1,0,1,2) and where 9 identifies the unlabelled data.
Using one hot encode
self._one_hot_encode(labels)

I get the following error: RuntimeError: index 1 is out of bounds for dimension 1 with size 1
due to
self.one_hot_labels = self.one_hot_labels.scatter(1, labels.unsqueeze(1), 1)

The error should raise from [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1], where I have 9 in the mapping setting equal index 9 to 1. It is unclear to me how to fix it, even after going through past questions and answers to similar problems (e.g., index 1 is out of bounds for dimension 0 with size 1).
The part of code involved in the error is the following:
def _one_hot_encode(self, labels):
    # Get the number of classes
    classes = torch.unique(labels)
    classes = classes[classes != 9] # unlabelled
    self.n_classes = classes.size(0)

    # One-hot encode labeled data instances and zero rows corresponding to unlabeled instances
    unlabeled_mask = (labels == 9)
    labels = labels.clone()  # defensive copying
    labels[unlabeled_mask] = 0
    self.one_hot_labels = torch.zeros((self.n_nodes, self.n_classes), dtype=torch.float)
    self.one_hot_labels = self.one_hot_labels.scatter(1, labels.unsqueeze(1), 1)
    self.one_hot_labels[unlabeled_mask, 0] = 0

    self.labeled_mask = ~unlabeled_mask

def fit(self, labels, max_iter, tol):

    self._one_hot_encode(labels)

    self.predictions = self.one_hot_labels.clone()
    prev_predictions = torch.zeros((self.n_nodes, self.n_classes), dtype=torch.float)

    for i in range(max_iter):
        # Stop iterations if the system is considered at a steady state
        variation = torch.abs(self.predictions - prev_predictions).sum().item()

        prev_predictions = self.predictions
        self._propagate()

Example of dataset:
ID  Target  Weight  Label   Score   Scale_Cat   Scale_num
0   A   D   65.1    1   87  Up  1
1   A   X   35.8    1   87  Up  1
2   B   C   34.7    1   37.5    Down    -2
3   B   P   33.4    1   37.5    Down    -2
4   C   B   33.1    1   37.5    Down    -2
5   S   X   21.4    0   12.5    NA  9

The source code I am using as reference is here: https://mybinder.org/v2/gh/thibaudmartinez/label-propagation/master?filepath=notebook.ipynb
Full track of the error:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-126-792a234f63dd> in <module>
      4 label_propagation = LabelPropagation(adj_matrix_t)
----> 6 label_propagation.fit(labels_t) # causing error
      7 label_propagation_output_labels = label_propagation.predict_classes()
      8

<ipython-input-115-54a7dbc30bd1> in fit(self, labels, max_iter, tol)
    100
    101     def fit(self, labels, max_iter=1000, tol=1e-3):
--> 102         super().fit(labels, max_iter, tol)
    103
    104 ## Label spreading

<ipython-input-115-54a7dbc30bd1> in fit(self, labels, max_iter, tol)
     58             Convergence tolerance: threshold to consider the system at steady state.
     59         """
---> 60         self._one_hot_encode(labels)
     61
     62         self.predictions = self.one_hot_labels.clone()

<ipython-input-115-54a7dbc30bd1> in _one_hot_encode(self, labels)
     42         labels[unlabeled_mask] = 0
     43         self.one_hot_labels = torch.zeros((self.n_nodes, self.n_classes), dtype=torch.float)
---> 44         self.one_hot_labels = self.one_hot_labels.scatter(1, labels.unsqueeze(1), 1)
     45         self.one_hot_labels[unlabeled_mask, 0] = 0
     46

RuntimeError: index 1 is out of bounds for dimension 1 with size 1

Output:
I ran through your notebook (I think you changed the 9 to -1 for things to run) and saw that for this part of the code:
# Learn with Label Propagation
label_propagation = LabelPropagation(adj_matrix_t)
print("Label Propagation: ", end="")
label_propagation.fit(labels_t)
label_propagation_output_labels = label_propagation.predict_classes()

Which eventually calls:
self.one_hot_labels = self.one_hot_labels.scatter(1, labels.unsqueeze(1), 1)

Is where things were going wrong.
Take a brief moment to read the pytorch manual on scatter here: torch Scatter and we learn that for scatter it's important to understand the dim, index, src and self matrixes. For one hot encoding, dim=1 or 0 doesn't matter and our src matrix is 1 (We'll look a little more into this later). You are now calling scatter on dimension 1 with an index matrix of [40,1] and a result(self) matrix of [40,5].
I see two issues here:

You are using the literal category dummy variables (-2,-1,0,1,2) as the encoding indexes in your index matrix. Which will lead scatter to search for these indices in the src matrix. This is where the index out of bounds in coming from
You mention that there are 6 classes of -2,-1,0,1,2 and 9 for unlabelled but you are one hot encoding on 5 classes. (Yes, I know you want the unlabeled class to be all zeros but that's a little difficult to achieve with scatter. I'll explain later).

So how do we fix this?
Issue 1: Let's start with a small example:
index = torch.tensor([[5],[0],[3],[5],[1],[4]]); print(index.shape); print(index)
result = torch.zeros(6, 6, dtype=src.dtype).scatter_(1, index, src); print(result.shape); print(result)

This will give us
torch.Size([6, 1])
tensor([[5],
        [0],
        [3],
        [5],
        [1],
        [4]])
torch.Size([6, 6])
tensor([[0, 0, 0, 0, 0, 1],
        [1, 0, 0, 0, 0, 0],
        [0, 0, 0, 1, 0, 0],
        [0, 0, 0, 0, 0, 1],
        [0, 1, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0]])

Index matrix is 6 observations with 1 observed value (category)
Self matrix is 6 observations with a 6 category one hot encoding vector
The way that scatter(dim=1) creates the self matrix is torch first checks the row (observation) and then changes the value of that row to the value of the value stored in the src matrix at the same row but at the column of the value stored in index.
self[i][index[i][j][k]][k] = src[i][j][k]

So in your case you were trying to apply the value of 1 into a row in self[40,1] at the column of index[0](which is equal to 1). Giving you the error in the question. Although I checked your notebook and the error is
index -1 is out of bounds for dimension 1 with size 5. They are both the same root cause.
Issue 2: One-hot-encoding
It is just easier to do complete one-hot instead of one-hot with cold encodings in this case. The reason being is that for one-hot with cold encodings, you need to create a 0 value in your src matrix for every unlabelled observation. Which is much more painful than just using a 1 for the src. Also reading this link: Is it valid to have full zeros for OHE? I think it makes more sense to use one-hot for every category.
So, for the second issue we just need to simply map the categories in the indexes of the result/self matrix. Since we have 6 categories we just need to map them into 0,1,2,3,4,5. A simple lambda function would do the trick. I used a random sampler to get my data labels from a class list as shown below: (I randomly created 40 observations from 6 classes)
classes = list([-2,-1,0,1,2,9])

labels = list()
for i in range(0,40):
    labels.append(list([(lambda x: x+2 if x !=9 else 5)(random.sample(classes,1)[0])]))

index_aka_labels = torch.tensor(labels)
print(index_aka_labels)
print(index_aka_labels.shape)
torch.zeros(40, 6, dtype=src.dtype).scatter_(1, index_aka_labels, 1)

Finally, we have achieved our desired result of OHE:
tensor([[0, 0, 0, 0, 0, 1],
        [0, 0, 1, 0, 0, 0],
        [0, 0, 0, 0, 1, 0],
        [0, 0, 0, 0, 1, 0],
        ... (40 observations)
        [0, 1, 0, 0, 0, 0],
        [0, 0, 0, 1, 0, 0],
        [1, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 1],


Instruction: How to add the last classification layer in EfficieNet pre-trained model in Pytorch?

Input:
I'm using the EfficientNet pre-trained model for my image classification project in Pytorch, and my purpose is to change the number of classes which is initially 1000 to 4.
However, for that when I try adding a model._fc layer, I keep on seeing this error "EfficientNet' object has no attribute 'classifier". Here is my code (Config.NUM_CLASSES = 4):
elif Config.MODEL_NAME == 'efficientnet-b3':

    from efficientnet_pytorch import EfficientNet
    model = EfficientNet.from_pretrained('efficientnet-b3')
    model._fc= torch.nn.Linear(in_features=model.classifier.in_features, **out_features=Config.NUM_CLASSES**, bias=True)

The situation is different when I add model._fc to the end of the Resnet part, it clearly changes the number of output classes to 4 in Resnet-18. Here is the code for that:
if Config.MODEL_NAME == 'resnet18': model = models.resnet50(pretrained=True) model.fc = torch.nn.Linear(in_features=model.fc.in_features, out_features=Config.NUM_CLASSES, bias=True)
The solution is available for TensorFlow and Keras, and I would really appreciate it if anyone could help me with that in PyTorch.
Regards,
Far

Output:
EfficentNet class doesn't have attribute classifier, you need to change in_features=model.classifier.in_features to in_features=model._fc.in_features.
import torchvision.models as models

NUM_CLASSES = 4

#EfficientNet
from efficientnet_pytorch import EfficientNet
efficientnet = EfficientNet.from_pretrained('efficientnet-b3')
efficientnet ._fc= torch.nn.Linear(in_features=efficientnet._fc.in_features, out_features=NUM_CLASSES, bias=True)

#mobilenet_v2
mobilenet = models.mobilenet_v2(pretrained=True)
mobilenet.classifier = nn.Sequential(nn.Dropout(p=0.2, inplace=False),
                  nn.Linear(in_features=mobilenet.classifier[1].in_features, out_features=NUM_CLASSES, bias=True))

#inception_v3
inception = models.inception_v3(pretrained=True)
inception.fc =  nn.Linear(in_features=inception.fc.in_features, out_features=NUM_CLASSES, bias=True)


Instruction: Tensorflow version of Pytorch Transforms

Input:
I have the following code that I use to prepare images before performing inference in a pytorch model:
def image_loader(transform, image_name):
    image = Image.open(image_name)
    #transform
    image = transform(image).float()
    image = torch.tensor(image)
    image = image.unsqueeze(0)
    return image

data_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

I've converted the model into a Tensorflow model, however, I'm unsure how I would do similar transformations to images before inference since there doesn't seem to be a tensorflow or keras equivalent. Any advice?

Output:
Here is some pointer, in pytorch you have
from torchvision import transforms
from PIL import Image
import torch

def image_loader(transform, image_name):
    image = Image.open(image_name).convert('RGB')
    image = transform(image).float()
    image = torch.tensor(image)
    image = image.unsqueeze(0)
    return image

data_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

# check: visualize
i = image_loader(data_transforms, '/content/1.png')
i.shape

plt.figure(figsize=(25,10))
subplot(121); imshow(np.array(i[0]).transpose(1, 2, 0));

And in tensorflow, you can achieve this as follows
def transform(image, mean, std):
    for channel in range(3):
        image[:, :, channel] = (image[:, :, channel] - mean[channel]) \
            / std[channel]
    return image

def image_loader(image_name):
    image = Image.open(image_name).convert('RGB')
    image = transform(np.array(image) / 255, mean=[0.485, 0.456,
                      0.406], std=[0.229, 0.224, 0.225])
    image = tf.cast(image, tf.float32)
    image = tf.expand_dims(image, 0)
    return image

# check: visualize
i = image_loader('/content/1.png')
i.shape

plt.figure(figsize=(25,10))
subplot(121); imshow(i[0]);

This should output the same. Note, in the second case, we define the transform function, from another OP, here, it's fine, however, you can also check tf. keras...Normalization, see this answer for details.


Instruction: PyTorch ImageFolder: Which Index Does a Folder Get?

Input:
I want to create a confusion matrix as it is shown here, though I use a different dataset than FashinMNIST. Concretely, I want the axes to contain names such as "banana", "orange", "cucumber", etc. I have a folder where images of bananas, etc. are stored, which I load with PyTorch's ImageFolder class. Now, PyTorch automatically assings the numbers 0 to 9 to my classes (I have 10 classes in total), but I'm not sure whether the label "banana" always gets the same number 0.
How do I find out, when using the ImageFolder class, which index belongs to which label (subfolder name)?

Output:
Class ImageFolder has attribute class_to_idx. Check the docs.
x = torchvision.datasets.ImageFolder(root=path, transform=transform)
print(x.class_to_idx)


Instruction: How can I see the first filters of a convolution layer

Input:
From this blog post, It was said that if you want to see kernel filter matrices directly, We can get the weight tensor and index into it accordingly. Now I want to see the first kernel of the first filters of my layer.
From the blog, It was mentioned that I can use this conv1.weight[1,1,:,:] How do use this for the below model architecture.
class Cifar10CnnModel(ImageClassificationBase):
    def __init__(self):
        super().__init__()
        self.network = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 64 x 16 x 16

            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 128 x 8 x 8

            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 256 x 4 x 4

            nn.Flatten(),
            nn.Linear(256*4*4, 1024),
            nn.ReLU(),
            nn.Linear(1024, 512),
            nn.ReLU(),
            nn.Linear(512, 10))

    def forward(self, xb):
        return self.network(xb)

Output:
If you create a model by:
model = Cifar10CnnModel()

then your weights will be stored in a sequential model and can be accessed by selecting the first element of the sequential model list:
model.network[0].weights


Instruction: Is there a way to run a function before the optimizer updates the weights?

Input:
I'm going through PyTorch tutorial and just learned about optimizer.step and how it makes an update to the network's parameters (here).
Is there a way to create a function that whenever theres a gradient updates to each learnable parameter (e.g weight), will take the weight value and the loss, and multiply that value by some percentage of that, say 90%?
So if the update should be:
w1 -= lr * loss_value = 1e-5 * 50
I want it to go through the function before the update and make it 1e-5 * 50 * 90%
def func(loss_value, percentage):
    return loss_value * percentage #new update should be w1 -= loss_value * percentage

Example model:
import torch
import torch.nn as nn
import torch.optim as optim

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.fc1 = nn.Linear(1, 5)
        self.fc2 = nn.Linear(5, 10)
        self.fc3 = nn.Linear(10, 1)

    def forward(self, x):
        x = self.fc1(x)
        x = torch.relu(x)
        x = torch.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Model()

opt = optim.Adam(net.parameters())
features = torch.rand((3,1))
opt.zero_grad()
out = net(features)
loss = torch.tensor(5) - torch.sum(out)
loss.backward()
# need to have the function change the value of the loss update before the optimizer?
opt.step()

Output:
I got this bit of code from https://discuss.pytorch.org/t/how-to-modify-the-gradient-manually/7483/2 and edited it slightly:
loss.backward()
for p in model.parameters():
    weights = p.data
    scales = def_scales(weights)
    p.grad *= scales  # or whatever other operation
optimizer.step()

This goes through every parameter in the model (between the loss.backward() and BEFORE the optimizer step) and adjusts its stored gradient BEFORE backprop is applied.
An example def_scales will look something like this (SUPER ugly), where vals are the compared parameter values, and scales are the desired loss scaling values:
def def_scales(weights,scales=[0.1,0.5,1,1],vals=[0,5,10,float('inf')]):
   out = torch.zeros_like(weights)
   for V,v in enumerate(vals[::-1]): #backwards because we're doing less than
      out[weights<=v] = scales[len(scales)-V-1] #might want to compare to abs
   return out


Instruction: RuntimeError: The size of tensor a (4144) must match the size of tensor b (256) at non-singleton dimension 3 site:stackoverflow.com

Input:
I am training a generator network with an image size (3, 256, 256). The network is as shown in the below
# Number of channels in the training images. For color images this is 3
nc = 3

# Size of z latent vector (i.e. size of generator input)
nz = 3

# Size of feature maps in generator
ngf = 64

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.main = nn.Sequential(
            # input is Z, going into a convolution
            nn.ConvTranspose2d( nz, ngf * 8, 4, 1, 0, bias=False),
            nn.BatchNorm2d(ngf * 8),
            nn.ReLU(True),
            # state size. (ngf*8) x 4 x 4
            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 4),
            nn.ReLU(True),
            # state size. (ngf*4) x 8 x 8
            nn.ConvTranspose2d( ngf * 4, ngf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 2),
            nn.ReLU(True),
            # state size. (ngf*2) x 16 x 16
            nn.ConvTranspose2d( ngf * 2, ngf, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf),
            nn.ReLU(True),
            # state size. (ngf) x 32 x 32
            nn.ConvTranspose2d( ngf, nc, 4, 2, 1, bias=False),
            nn.Tanh()
            # state size. (nc) x 64 x 64
        )

    def forward(self, input):
        return self.main(input)

net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()
out_avg = None
last_net = None
psrn_noisy_last = 0
loss = []
psnr_noise = []
psnr_ground = []

i = 0
def closure():

    global i, out_avg, psrn_noisy_last, last_net, net_input, loss

    if reg_noise_std > 0:
        net_input = net_input_saved + (noise.normal_() * reg_noise_std) #changing the input to the netwok

    out = net(net_input)

    # Smoothing
    if out_avg is None:
        out_avg = out.detach()
    else:
        out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight) # calculating average network output

    total_loss = mse(out, img_noisy_torch)
    total_loss.backward()

    loss.append(total_loss.item())

    # caculating psrn
    psrn_noisy = compare_psnr(img_noisy_np, out.detach().cpu().numpy()[0]) # comparing psnr for the output image and the actual noisy image
    psrn_gt    = compare_psnr(img_noisy_np, out.detach().cpu().numpy()[0]) # comparing psnr for the output image and the original image
    psrn_gt_sm = compare_psnr(img_np, out_avg.detach().cpu().numpy()[0]) # comparing psnr for the output average and the original image

    psnr_noise.append(psrn_noisy)
    psnr_ground.append(psrn_gt)

    if  PLOT and i % show_every == 0:
        out_np = torch_to_np(out)
        # plotting the output image along the average image calculated
        print(f'\n\nAfter {i} iterations: ')
        print ('Iteration %05d    Loss %f   PSNR_noisy: %f   PSRN_gt: %f   PSNR_gt_sm: %f' % (i, total_loss.item(), psrn_noisy, psrn_gt, psrn_gt_sm), '\r', end='\n')
        plot_image_grid([np.clip(out_np, 0, 1),
                         np.clip(torch_to_np(out_avg), 0, 1)], factor=figsize, nrow=1)

    # Backtracking
    if i % show_every:
        if psrn_noisy - psrn_noisy_last < -5:
            print('Falling back to previous checkpoint.')

            for new_param, net_param in zip(last_net, net.parameters()):
                net_param.data.copy_(new_param.cuda())

            return total_loss*0
        else:
            last_net = [x.detach().cpu() for x in net.parameters()]
            psrn_noisy_last = psrn_noisy

    i += 1

    return total_loss

p = get_params(OPT_OVER, net, net_input)
optimize(OPTIMIZER, p, closure, LR, num_iter)

when i try to train i am getting error
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:528: UserWarning: Using a target size (torch.Size([1, 3, 256, 256])) that is different to the input size (torch.Size([1, 3, 4144, 4144])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-83-ed04cb48a6ab> in <module>()
     67
     68 p = get_params(OPT_OVER, net, net_input)
---> 69 optimize(OPTIMIZER, p, closure, LR, num_iter)

5 frames
/content/utils/common_utils.py in optimize(optimizer_type, parameters, closure, LR, num_iter)
    227         for j in range(num_iter):
    228             optimizer.zero_grad()
--> 229             closure()
    230             optimizer.step()
    231     else:

<ipython-input-83-ed04cb48a6ab> in closure()
     25         out_avg = out_avg * exp_weight + out.detach() * (1 - exp_weight) # calculating average network output
     26
---> 27     total_loss = mse(out, img_noisy_torch)
     28     total_loss.backward()
     29

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1049         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1050                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051             return forward_call(*input, **kwargs)
   1052         # Do not call functions when jit is used
   1053         full_backward_hooks, non_full_backward_hooks = [], []

/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py in forward(self, input, target)
    526
    527     def forward(self, input: Tensor, target: Tensor) -> Tensor:
--> 528         return F.mse_loss(input, target, reduction=self.reduction)
    529
    530

/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in mse_loss(input, target, size_average, reduce, reduction)
   3087         reduction = _Reduction.legacy_get_string(size_average, reduce)
   3088
-> 3089     expanded_input, expanded_target = torch.broadcast_tensors(input, target)
   3090     return torch._C._nn.mse_loss(expanded_input, expanded_target, _Reduction.get_enum(reduction))
   3091

/usr/local/lib/python3.7/dist-packages/torch/functional.py in broadcast_tensors(*tensors)
     71     if has_torch_function(tensors):
     72         return handle_torch_function(broadcast_tensors, tensors, *tensors)
---> 73     return _VF.broadcast_tensors(tensors)  # type: ignore[attr-defined]
     74
     75

RuntimeError: The size of tensor a (4144) must match the size of tensor b (256) at non-singleton dimension 3

I understood this error is coming from mismatch of sizes in tensors, but i was unable to rectify the error.

I am providing noise z of size torch.Size([1, 3, 256, 256]) same as the dimensions of the input image to the network but i am getting error.

Output: I think the problem is the net_input before going through the model is not size=(1,3,256,256) instead (1,3,4144,4144). Try resize net_input.


Instruction: torch virtual env not working python 3.7 , what am i doing wrong?

Input:
so i installed pytorch using conda into a virtual env while referring to this video
https://www.youtube.com/watch?v=vBfM5l9VK5c
i have activated the env
now inside jupyter notebook i run
import torch
print(torch.__version__)

and it works but when ever i run this in .py file and run it through terminal it gives me this error
import torch
ModuleNotFoundError: No module named 'torch'

if I try to pip install pytorch it says
 Requirement already satisfied: torchvision in c:\users\kiit\anaconda3\envs\torch\lib\site-packages (0.10.0)
Requirement already satisfied: numpy in c:\users\kiit\anaconda3\envs\torch\lib\site-packages (from torchvision) (1.20.3)
Requirement already satisfied: torch==1.9.0 in c:\users\kiit\anaconda3\envs\torch\lib\site-packages (from torchvision) (1.9.0)
Requirement already satisfied: pillow>=5.3.0 in c:\users\kiit\anaconda3\envs\torch\lib\site-packages (from torchvision) (8.2.0)
Requirement already satisfied: typing_extensions in c:\users\kiit\anaconda3\envs\torch\lib\site-packages (from torch==1.9.0->torchvision) (3.7.4.3)

so what is going on exactly??

Output:
You might have more than one python versions installed on your system i.e One with conda and one separately.
In order to check that, you can go to control panel --> Programs and features. There you can figure out how many python installations are present on your system. Delete the one you dont need and your problem will be resolved.


Instruction: How to add mask to loss function in PyTorch

Input:
My PyTorch model outputs two images: op and psuedo-opand I wish to backpropagate at only those pixels where loss(op_i,gt_i)<loss(psuedo-op_i,gt_i), where i is used to index pixels. Clearly, loss(op,gt).backward() cannot achieve this. So how to do?
Have some rough solution in mind as follows:
def loss(pred, target):
       """ shape of pred/target is BatchxChannelxHeightxWidth, where Channel=1 """

        abs_diff = torch.abs(target - pred)
        l1_loss = abs_diff.mean(1, True)

        return l1_loss

o1 = loss(op,gt)
o2 = loss(psuedo-op,gt)
o = torch.cat((o1,o2),dim=1)

value, idx = torch.min(o,dim=1)

NOW SOMEHOW USE IDX TO GENERATE MASK AND SELECTIVE BACKPROPAGATION

Any other solution would also work if it allows me to backpropagate on o1 but for only those pixels where o1<o2.

Output:
You can use the relu function for this purpose I think. Since you need to backprop only on o1, you first need to detach the loss o2. And there is also a minus sign to correct the sign of the gradient.
# This diff is 0 when o1 > o2, equal to o2-o1 otherwise
o_diff = nn.functional.relu(o2.detach()-o1)
# gradient of (-relu(b-x)) is 0 if b-x < 0, 1 otherwise
(-o_diff).sum().backward()

Here, using the relu as a kind of conditional on the sign of o2-o1 makes it very easy to to void gradients for coefficients with minus sign
I need to emphasize that since o2 is detached from the graph it is a constant with respect to your network, so it does not affect the gradient and thus this operation achieves what you need : it is basically backpropagating d/dx(-relu(b-o1(x)) which is 0 if b < o1(x) and d/dx(o1(x)) otherwise (where b = o2 is constant).


Instruction: PyTorch in-place operator issue in NumPy conversion

Input:
I converted a PyTorch Tensor into a NumPy ndarray, and as it's shown below 'a' and 'b' both have different ids so they point to different memory locations.
I know in PyTorch underscore indicates in-place operation so a.add_(1) does not change the memory location of a, but why does it change the content of the array b (due to the difference in id)?
import torch
a = torch.ones(5)
b = a.numpy()
a.add_(1)

print(id(a), id(b))
print(a)
print(b)

Results:
139851293933696 139851293925904
tensor([2., 2., 2., 2., 2.])
[2. 2. 2. 2. 2.]

PyTorch documentation:

Converting a torch Tensor to a NumPy array and vice versa is a breeze.
The torch Tensor and NumPy array will share their underlying memory
locations, and changing one will change the other.

But why?
(They have different IDs so must be independent) :(

Output:
If you use numpy.ndarray.__array_interface__ and torch.Tensor.data_ptr(), you can find the memory location of their data_array.
a = torch.ones(5)
b = a.numpy()
a.add_(1)

print(b.__array_interface__)
# > {'data': (1848226926144, False),
#    'strides': None,
#    'descr': [('', '<f4')],
#    'typestr': '<f4',
#    'shape': (5,),
#    'version': 3}

print(a.data_ptr())
# > 1848226926144

a.data_ptr() == b.__array_interface__["data"][0]
# > True

Both the array and the tensor share the same data_array.
We can reasonably conclude that the method torch.Tensor.numpy() creates a numpy array from the Tensor by passing it the reference to its data_array.
b[0] = 4
print(a)
# > tensor([4., 2., 2., 2., 2.])