Instruction: Cannot create the calibration cache for the QAT model in tensorRT

Input:
I've trained a quantized model (with help of quantized-aware-training method in pytorch). I want to create the calibration cache to do inference in INT8 mode by TensorRT. When create calib cache, I get the following warning and the cache is not created:
[03/06/2022-08:14:07] [TRT] [W] Calibrator won't be used in explicit precision mode. Use quantization aware training to generate network with Quantize/Dequantize nodes.
[03/06/2022-08:14:11] [TRT] [W] Some weights are outside of int8_t range and will be clipped to int8_t range.
[03/06/2022-08:14:11] [TRT] [W] Some weights are outside of int8_t range and will be clipped to int8_t range.
[03/06/2022-08:14:11] [TRT] [W] Some weights are outside of int8_t range and will be clipped to int8_t range.
[03/06/2022-08:14:11] [TRT] [W] Some weights are outside of int8_t range and will be clipped to int8_t range.

I've trained the model accordingly and converted to ONNX:
import os
import sys
import argparse
import warnings
import collections

import torch
import torch.utils.data
from torch import nn

from tqdm import tqdm
import torchvision
from torchvision import transforms
from torch.hub import load_state_dict_from_url
from pytorch_quantization import nn as quant_nn
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor
from pytorch_quantization import quant_modules
import onnxruntime
import numpy as np
import models
import kornia
from prettytable import PrettyTable

from PIL import Image, ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

def get_parser():
    """
    Creates an argument parser.
    """
    parser = argparse.ArgumentParser(description='Classification quantization flow script')

    parser.add_argument('--data-dir', '-d', type=str, help='input data folder', required=True)
    parser.add_argument('--model-name', '-m', default='', help='model name: default resnet50')
    parser.add_argument('--disable-pcq', '-dpcq', action="store_true", help='disable per-channel quantization for weights')
    parser.add_argument('--out-dir', '-o', default='/tmp', help='output folder: default /tmp')
    parser.add_argument('--print-freq', '-pf', type=int, default=20, help='evaluation print frequency: default 20')
    parser.add_argument('--threshold', '-t', type=float, default=-1.0, help='top1 accuracy threshold (less than 0.0 means no comparison): default -1.0')

    parser.add_argument('--batch-size-train', type=int, default=8, help='batch size for training: default 128')
    parser.add_argument('--batch-size-test', type=int, default=8, help='batch size for testing: default 128')
    parser.add_argument('--batch-size-onnx', type=int, default=20, help='batch size for onnx: default 1')

    parser.add_argument('--seed', type=int, default=12345, help='random seed: default 12345')

    checkpoint = parser.add_mutually_exclusive_group(required=True)
    checkpoint.add_argument('--ckpt-path', default='', type=str, required=False,
                            help='path to latest checkpoint (default: none)')
    checkpoint.add_argument('--ckpt-url', default='', type=str, required=False,
                            help='url to latest checkpoint (default: none)')
    checkpoint.add_argument('--pretrained', action="store_true")

    parser.add_argument('--num-calib-batch', default=8, type=int,
                        help='Number of batches for calibration. 0 will disable calibration. (default: 4)')
    parser.add_argument('--num-finetune-epochs', default=0, type=int,
                        help='Number of epochs to fine tune. 0 will disable fine tune. (default: 0)')
    parser.add_argument('--calibrator', type=str, choices=["max", "histogram"], default="max")
    parser.add_argument('--percentile', nargs='+', type=float, default=[99.9, 99.99, 99.999, 99.9999])
    parser.add_argument('--sensitivity', action="store_true", help="Build sensitivity profile")
    parser.add_argument('--evaluate-onnx', action="store_true", help="Evaluate exported ONNX")

    return parser

def prepare_model(
        model_name,
        num_class,
        data_dir,
        per_channel_quantization,
        batch_size_train,
        batch_size_test,
        batch_size_onnx,
        calibrator,
        pretrained,
        ckpt_path,
        ckpt_url=None):

    ## Initialize quantization, model and data loaders
    if per_channel_quantization:
        print('<<<<<<< Per channel qaunt >>>>>>>>')
        quant_desc_input = QuantDescriptor(calib_method=calibrator)
        quant_nn.QuantConv2d.set_default_quant_desc_input(quant_desc_input)
        quant_nn.QuantLinear.set_default_quant_desc_input(quant_desc_input)
    else:
        ## Force per tensor quantization for onnx runtime
        print('<<<<<<< Per tensor qaunt >>>>>>>>')
        quant_desc_input = QuantDescriptor(calib_method=calibrator, axis=None)
        quant_nn.QuantConv2d.set_default_quant_desc_input(quant_desc_input)
        quant_nn.QuantConvTranspose2d.set_default_quant_desc_input(quant_desc_input)
        quant_nn.QuantLinear.set_default_quant_desc_input(quant_desc_input)

        quant_desc_weight = QuantDescriptor(calib_method=calibrator, axis=None)
        quant_nn.QuantConv2d.set_default_quant_desc_weight(quant_desc_weight)
        quant_nn.QuantConvTranspose2d.set_default_quant_desc_weight(quant_desc_weight)
        quant_nn.QuantLinear.set_default_quant_desc_weight(quant_desc_weight)

    if model_name in models.__dict__:
        model = models.__dict__[model_name](pretrained=pretrained, quantize=True)
        num_feats = model.fc.in_features
        model.fc = nn.Linear(num_feats, num_class)

    else:
        print('Model is not available....downloading....')
        quant_modules.initialize()
        model = torchvision.models.__dict__[model_name](pretrained=pretrained)
        if 'resnet' in model_name:
            num_feats = model.fc.in_features
            model.fc = nn.Linear(num_feats, num_class)
        if 'densenet' in model_name:
            num_feats =  model.classifier.in_features
            model.classifier = nn.Linear(num_feats, num_class)
        quant_modules.deactivate()

    if not pretrained:
        if ckpt_path:
            model = torch.load(ckpt_path)
        else:
            model = load_state_dict_from_url(ckpt_url)
        if 'state_dict' in model.keys():
            model = model['state_dict']
        elif 'model' in model.keys():
            model = model['model']
        # model.load_state_dict(checkpoint)
    model.eval()
    model.cuda()
    print(model)

    ## Prepare the data loaders
    traindir = os.path.join(data_dir, 'train')
    valdir = os.path.join(data_dir, 'test')
    _args = collections.namedtuple("mock_args", ["model", "distributed", "cache_dataset"])
    dataset, dataset_test, train_sampler, test_sampler = load_data(
        traindir, valdir, _args(model=model_name, distributed=False, cache_dataset=False))

    data_loader_train = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size_train,
        sampler=train_sampler, num_workers=4, pin_memory=True)

    data_loader_test = torch.utils.data.DataLoader(
        dataset_test, batch_size=batch_size_test,
        sampler=test_sampler, num_workers=4, pin_memory=True)

    data_loader_onnx = torch.utils.data.DataLoader(
        dataset_test, batch_size=batch_size_onnx,
        sampler=test_sampler, num_workers=4, pin_memory=True)

    return model, data_loader_train, data_loader_test, data_loader_onnx

def main(cmdline_args):
    parser = get_parser()
    args = parser.parse_args(cmdline_args)
    print(parser.description)
    print(args)

    torch.manual_seed(args.seed)
    np.random.seed(args.seed)

    args.model_name = 'resnet34'
    args.data_dir = '/home/Dataset/'
    args.disable_pcq = True #set true to disbale
    args.batch_size_train = 8
    args.batch_size_test = 8
    args.batch_size_onnx = 8
    args.calibrator = 'max'
    args.pretrained = True
    args.ckpt_path = ''
    args.ckpt_url = ''
    args.num_class = 5

    ## Prepare the pretrained model and data loaders
    model, data_loader_train, data_loader_test, data_loader_onnx = prepare_model(
        args.model_name,
        args.num_class,
        args.data_dir,
        not args.disable_pcq,
        args.batch_size_train,
        args.batch_size_test,
        args.batch_size_onnx,
        args.calibrator,
        args.pretrained,
        args.ckpt_path,
        args.ckpt_url)

    kwargs = {"alpha": 0.75, "gamma": 2.0, "reduction": 'mean'}
    criterion = kornia.losses.FocalLoss(**kwargs)

    ## Calibrate the model
    with torch.no_grad():
        calibrate_model(
            model=model,
            model_name=args.model_name,
            data_loader=data_loader_train,
            num_calib_batch=args.num_calib_batch,
            calibrator=args.calibrator,
            hist_percentile=args.percentile,
            out_dir=args.out_dir)

    ## Build sensitivy profile
    if args.sensitivity:
        build_sensitivity_profile(model, criterion, data_loader_test)

    kwargs = {"alpha": 0.75, "gamma": 3.0, "reduction": 'mean'}
    criterion = kornia.losses.FocalLoss(**kwargs)

    optimizer = torch.optim.SGD(model.parameters(), lr=0.0001)
    lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.num_finetune_epochs)
    for epoch in range(args.num_finetune_epochs):
        # Training a single epch
        train_one_epoch(model, criterion, optimizer, data_loader_train, "cuda", epoch, 100)
        lr_scheduler.step()

    if args.num_finetune_epochs > 0:
        ## Evaluate after finetuning
        with torch.no_grad():
            print('Finetune evaluation:')
            top1_finetuned = evaluate(model, criterion, data_loader_test, device="cuda")
    else:
        top1_finetuned = -1.0

    ## Export to ONNX
    onnx_filename = args.out_dir + '/' + args.model_name + ".onnx"
    top1_onnx = -1.0
    if export_onnx(model, onnx_filename, args.batch_size_onnx, not args.disable_pcq) and args.evaluate_onnx:
        ## Validate ONNX and evaluate
        top1_onnx = evaluate_onnx(onnx_filename, data_loader_onnx, criterion, args.print_freq)

    ## Print summary
    print("Accuracy summary:")
    table = PrettyTable(['Stage','Top1'])
    table.align['Stage'] = "l"
    table.add_row( [ 'Finetuned',   "{:.2f}".format(top1_finetuned) ] )
    table.add_row( [ 'ONNX',        "{:.2f}".format(top1_onnx) ] )
    print(table)

    ## Compare results
    if args.threshold >= 0.0:
        if args.evaluate_onnx and top1_onnx < 0.0:
            print("Failed to export/evaluate ONNX!")
            return 1
        if args.num_finetune_epochs > 0:
            if top1_finetuned >= (top1_onnx - args.threshold):
                print("Accuracy threshold was met!")
            else:
                print("Accuracy threshold was missed!")
                return 1

    return 0

def evaluate_onnx(onnx_filename, data_loader, criterion, print_freq):

    print("Loading ONNX file: ", onnx_filename)
    ort_session = onnxruntime.InferenceSession(onnx_filename)
    with torch.no_grad():
        metric_logger = utils.MetricLogger(delimiter="  ")
        header = 'Test:'
        with torch.no_grad():
            for image, target in metric_logger.log_every(data_loader, print_freq, header):
                image = image.to("cpu", non_blocking=True)
                image_data = np.array(image)
                input_data = image_data

                # run the data through onnx runtime instead of torch model
                input_name = ort_session.get_inputs()[0].name
                raw_result = ort_session.run([], {input_name: input_data})
                output = torch.tensor((raw_result[0]))

                loss = criterion(output, target)
                acc1, acc5 = utils.accuracy(output, target, topk=(1, 5))
                batch_size = image.shape[0]
                metric_logger.update(loss=loss.item())
                metric_logger.meters['acc1'].update(acc1.item(), n=batch_size)
                metric_logger.meters['acc5'].update(acc5.item(), n=batch_size)
        # gather the stats from all processes
        metric_logger.synchronize_between_processes()

        print('  ONNXRuntime: Acc@1 {top1.global_avg:.3f} Acc@5 {top5.global_avg:.3f}'
            .format(top1=metric_logger.acc1, top5=metric_logger.acc5))
        return metric_logger.acc1.global_avg

def export_onnx(model, onnx_filename, batch_onnx, per_channel_quantization):
    model.eval()
    quant_nn.TensorQuantizer.use_fb_fake_quant = True # We have to shift to pytorch's fake quant ops before exporting the model to ONNX

    if per_channel_quantization:
        opset_version = 13
    else:
        opset_version = 12

    # Export ONNX for multiple batch sizes
    print("Creating ONNX file: " + onnx_filename)
    dummy_input = torch.randn(batch_onnx, 3, 224, 224, device='cuda') #TODO: switch input dims by model
    input_names = ['input']

    if 'resnet' in onnx_filename:
        print('Changing last layer of resnet...')
        output_names = ['Linear[fc]']  ### ResNet34
    if 'densenet' in onnx_filename:
        print('Changing last layer of densenet...')
        output_names = ['Linear[classifier]'] #### DenseNet
    dynamic_axes = {'input': {0: 'batch_size'}}

    try:
        torch.onnx.export(model, dummy_input, onnx_filename, input_names=input_names,
                          export_params=True, output_names=output_names, opset_version=opset_version,
                          dynamic_axes=dynamic_axes, verbose=True, enable_onnx_checker=False, do_constant_folding=True)

    except ValueError:
        warnings.warn(UserWarning("Per-channel quantization is not yet supported in Pytorch/ONNX RT (requires ONNX opset 13)"))
        print("Failed to export to ONNX")
        return False

    return True

def calibrate_model(model, model_name, data_loader, num_calib_batch, calibrator, hist_percentile, out_dir):

    if num_calib_batch > 0:
        print("Calibrating model")
        with torch.no_grad():
            collect_stats(model, data_loader, num_calib_batch)

        if not calibrator == "histogram":
            compute_amax(model, method="max")
            calib_output = os.path.join(
                out_dir,
                F"{model_name}-max-{num_calib_batch*data_loader.batch_size}.pth")
            # torch.save(model.state_dict(), calib_output) # Just weights
            torch.save(model, calib_output) # whole model
        else:
            for percentile in hist_percentile:
                print(F"{percentile} percentile calibration")
                compute_amax(model, method="percentile")
                calib_output = os.path.join(
                    out_dir,
                    F"{model_name}-percentile-{percentile}-{num_calib_batch*data_loader.batch_size}.pth")
                torch.save(model, calib_output) # whole model

            for method in ["mse", "entropy"]:
                print(F"{method} calibration")
                compute_amax(model, method=method)
                calib_output = os.path.join(
                    out_dir,
                    F"{model_name}-{method}-{num_calib_batch*data_loader.batch_size}.pth")
                # torch.save(model.state_dict(), calib_output)
                torch.save(model, calib_output)

def collect_stats(model, data_loader, num_batches):
    # Enable calibrators
    for name, module in model.named_modules():
        if isinstance(module, quant_nn.TensorQuantizer):
            if module._calibrator is not None:
                module.disable_quant()
                module.enable_calib()
            else:
                module.disable()

    # Feed data to the network for collecting stats
    for i, (image, _) in tqdm(enumerate(data_loader), total=num_batches):
        model(image.cuda())
        if i >= num_batches:
            break

    # Disable calibrators
    for name, module in model.named_modules():
        if isinstance(module, quant_nn.TensorQuantizer):
            if module._calibrator is not None:
                module.enable_quant()
                module.disable_calib()
            else:
                module.enable()

def compute_amax(model, **kwargs):
    # Load calib result
    for name, module in model.named_modules():
        if isinstance(module, quant_nn.TensorQuantizer):
            if module._calibrator is not None:
                if isinstance(module._calibrator, calib.MaxCalibrator):
                    module.load_calib_amax()
                else:
                    module.load_calib_amax(**kwargs)
            print(F"{name:40}: {module}")
    model.cuda()

def build_sensitivity_profile(model, criterion, data_loader_test):
    quant_layer_names = []
    for name, module in model.named_modules():
        if name.endswith("_quantizer"):
            module.disable()
            layer_name = name.replace("._input_quantizer", "").replace("._weight_quantizer", "")
            if layer_name not in quant_layer_names:
                quant_layer_names.append(layer_name)
    for i, quant_layer in enumerate(quant_layer_names):
        print("Enable", quant_layer)
        for name, module in model.named_modules():
            if name.endswith("_quantizer") and quant_layer in name:
                module.enable()
                print(F"{name:40}: {module}")
        with torch.no_grad():
            evaluate(model, criterion, data_loader_test, device="cuda")
        for name, module in model.named_modules():
            if name.endswith("_quantizer") and quant_layer in name:
                module.disable()
                print(F"{name:40}: {module}")

if __name__ == '__main__':
    res = main(sys.argv[1:])
    exit(res)

More info regarding system:
TensorRT == 8.2
Pytorch == 1.9.0+cu111
Torchvision == 0.10.0+cu111
ONNX == 1.9.0
ONNXRuntime == 1.8.1
pycuda == 2021

Output: If the ONNX model has Q/DQ nodes in it, you may not need calibration cache because quantization parameters such as scale and zero point are included in the Q/DQ nodes. You can run the Q/DQ ONNX model directly in TensorRT execution provider in OnnxRuntime (>= v1.9.0).


Instruction: The pytorch training model cannot be created successfully

Input:
I would like to do a neural network for regression analysis using optuna based on this site.
I would like to create a model with two 1D data as input and one 1D data as output in batch learning.
x is the training data and y is the teacher data.
class Model(nn.Module):
    # コンストラクタ(インスタンス生成時の初期化)
    def __init__(self,trial, mid_units1, mid_units2):
        super(Model, self).__init__()
        self.linear1 = nn.Linear(2, mid_units1)
        self.bn1 = nn.BatchNorm1d(mid_units1)
        self.linear2 = nn.Linear(mid_units1, mid_units2)
        self.bn2 = nn.BatchNorm1d(mid_units2)
        self.linear3 = nn.Linear(mid_units2, 1)
        self.activation = trial_activation(trial)

    def forward(self, x):
        x = self.linear1(x)
        x = self.bn1(x)
        x = self.activation(x)
        x = self.linear2(x)

device = "cuda" if torch.cuda.is_available() else "cpu"

EPOCH = 100
x = torch.from_numpy(a[0].astype(np.float32)).to(device)
y = torch.from_numpy(a[1].astype(np.float32)).to(device)

def train_epoch(model, optimizer, criterion):
    model.train()
    optimizer.zero_grad()    # 勾配情報を0に初期化
    y_pred = model(x)                                               # 予測
    loss = criterion(y_pred.reshape(y.shape), y)          # 損失を計算(shapeを揃える)
    loss.backward()                                                       # 勾配の計算
    optimizer.step()                                                      # 勾配の更新
    return loss.item()

def trial_activation(trial):
    activation_names = ['ReLU','logsigmoid']
    activation_name = trial.suggest_categorical('activation', activation_names)
    if activation_name == activation_names[0]:
        activation = F.relu
    else:
        activation = F.logsigmoid
    return activation

def objective(trial):
    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    # 中間層のユニット数の試行
    mid_units1 = int(trial.suggest_discrete_uniform("mid_units1", 1024*2,1024*4, 64*2))
    mid_units2 = int(trial.suggest_discrete_uniform("mid_units2", 1024, 1024*2, 64*2))

    net = Model(trial, mid_units1, mid_units2).to(device)

    criterion = nn.MSELoss()
    # 最適化手法の試行
    optimizer = trial_optimizer(trial, net)
    train_loss = 0
    for epoch in range(EPOCH):
        train_loss = train_epoch(net, optimizer, criterion, device)
    torch.save(net.state_dict(), str(trial.number) + "new1.pth")
    return train_loss

strage_name = "a.sql"
study_name = 'a'

study = optuna.create_study(
    study_name = study_name,
    storage='sqlite:///'  + strage_name,
    load_if_exists=True,
    direction='minimize')
TRIAL_SIZE = 100

study.optimize(objective, n_trials=TRIAL_SIZE)

error message
---> 28     loss = criterion(y_pred.reshape(y.shape), y)          # 損失を計算(shapeを揃える)
     29     loss.backward()                                                       # 勾配の計算
     30     optimizer.step()                                                      # 勾配の更新

AttributeError: 'NoneType' object has no attribute 'reshape'

Because of the above error, I checked the value of y_pred and found it to be None.
    model.train()
    optimizer.zero_grad()

I am thinking that these two lines may be wrong, but I don't know how to solve this problem.

Output:
With PyTorch, when you call y_pred = model(x) that will call the forward function which is defined in the Model class.
So, y_pred  will get the result of the forward function, in your case, it returns nothing, that's why you get a None value. You can change the forward function as below:
    def forward(self, x):
        x = self.linear1(x)
        x = self.bn1(x)
        x = self.activation(x)
        x = self.linear2(x)
        return x


Instruction: Rounding only specific entries of torch.tensor()

Input:
Using torch.round() is it possible to eventually round specific entries of a tensor? Example:
tensor([ 8.5040e+00,  7.3818e+01,  5.2922e+00, -1.8912e-01,  5.4389e-01,
        -3.6032e-03,  4.5763e-01, -2.7471e-02])

Desired output:
tensor([ 9.,  74.,  5., 0.,  5.4389e-01,
        -3.6032e-03,  4.5763e-01, -2.7471e-02])

(Only first 4 rounded)

Output:
you can do as follow
a[:4]=torch.round(a[:4])


Instruction: I want to increase the frequency of each element of an array with particular amount?

Input:
This is sample array;

I want it increase the frequency of each element by constant (taking 4 here);
[64,64,64,64, 45, 45, 45, 45, 56, 56, 56, 56, 67, 67, 67, 67, 78, 78, 78, 78, 12, 12, 12, 12, 112, 112, 112, 112, 232, 232, 232, 232]
Can anyone help me with this? Please it should not be hardcoded

Output:
You can use np.repeat to achieve this:
>>> a.repeat(4)
array([ 64,  64,  64,  64,  45,  45,  45,  45,  56,  56,  56,  56,  67,
        67,  67,  67,  78,  78,  78,  78,  12,  12,  12,  12, 112, 112,
       112, 112, 232, 232, 232, 232])


Instruction: Installing PyTorch in Google Colabs

Input:
I am currently trying to run my code in Google Colabs and for that I need PyTorch. This is the feedback I got.
> ValueError                                Traceback (most recent call last)
<ipython-input-26-3d0da87b83cc> in <module>()
     13 #from sentence_transformers import SentenceTransformer
     14 get_ipython().system('pip install torch')
---> 15 import torch
     16 get_ipython().system('pip install transformers')
     17 from transformers import BertTokenizer, BertModel
>
> /usr/local/lib/python3.7/dist-packages/torch/__init__.py in <module>()
>     195     if USE_GLOBAL_DEPS:
>     196         _load_global_deps()
> --> 197     from torch._C import *  # noqa: F403
>     198
>     199 # Appease the type checker; ordinarily this binding is inserted by the
>
> ValueError: module functions cannot set METH_CLASS or METH_STATIC

I read that it could have to do with Numpy and tried another version, which also failed.

Output:
Remove the old numpy and run the following code.
pip install -U numpy

Found the solution here.


Instruction: pytorch tensor of tensors to a tensor

Input:
When I print a torch tensor, I get the below output. How could I get that tensor without [] for the inner elements?
I printed the type of the first element and it returns <class 'torch.Tensor'> So this tensor seems to be a tensor of tensors...How could I convert it to a tensor of numbers?
tensor([[-5.6117e-01],
        [ 3.5726e-01],
        [-2.5853e-01],
        [-4.8641e-01],
        [-1.0581e-01],
        [-1.8322e-01],
        [-1.2732e+00],
        [-5.9760e-02],
        [ 1.2819e-01],
        [ 6.3894e-02],
        [-9.1817e-01],
        [-1.6539e-01],
        [-1.1471e+00],
        [ 1.9666e-01],
        [-6.3297e-01],
        [-4.0876e-01],
        [-2.4590e-02],
        [ 2.7065e-01],
        [ 3.5308e-01],
        [-4.6348e-01],
        [-4.1755e-01],
        [-1.1554e-01],
        [-4.2062e-01],
        [ 1.4067e-01],
        [-2.9788e-01],
        [-7.4582e-02],
        [-5.3751e-01],
        [ 1.1344e-01],
        [-2.6100e-01],
        [ 2.6951e-02],
        [-5.0437e-02],
        [-1.9163e-01],
        [-3.3893e-02],
        [-5.9640e-01],
        [-1.1574e-01],
        [ 1.4613e-01],
        [ 1.2263e-01],
        [-1.5566e-01],
        [ 1.4740e-01],
        [-9.9924e-01],
        [ 2.0878e-01],
        [-2.0074e-01],
        [ 7.8383e-02],
        [ 7.4679e-02],
        [-5.8065e-01],
        [ 6.7777e-01],
        [ 5.9879e-01],
        [ 6.6301e-01],
        [-4.7051e-01],
        [-2.5468e-01],
        [-2.7382e-01],
        [ 1.7585e-01],
        [ 3.6151e-01],
        [-9.2532e-01],
        [-1.6999e-01],
        [ 8.4971e-02],
        [-6.6083e-01],
        [-3.1204e-02],
        [ 6.3712e-01],
        [-5.8580e-02],
        [-7.7901e-04],
        [-4.6792e-01],
        [ 1.0796e-01],
        [ 7.8766e-01],
        [ 1.6809e-01],
        [-7.0058e-01],
        [-2.9299e-01],
        [-8.2735e-02],
        [ 2.0875e-01],
        [-2.9426e-01],
        [-7.6748e-02],
        [-1.5762e-01],
        [-5.7432e-01],
        [-5.2042e-01],
        [-1.5152e-01],
        [ 1.4119e+00],
        [-1.5752e-01],
        [-3.0565e-01],
        [-5.1378e-01],
        [-5.8924e-01],
        [-1.0163e+00],
        [-2.2021e-01],
        [ 2.9112e-02],
        [ 1.8521e-01],
        [ 6.2814e-01],
        [-6.8793e-01],
        [ 2.1395e-02],
        [ 5.7168e-01],
        [ 9.0977e-01],
        [ 3.8899e-01],
        [ 3.0209e-01],
        [ 2.4655e-01],
        [-1.1688e-01],
        [-5.9835e-02],
        [ 3.6426e-02],
        [-5.2782e-01],
        [ 1.4604e+00],
        [ 2.9685e-01],
        [-2.4077e-01],
        [ 1.0163e+00],
        [ 6.9770e-01],
        [-2.6183e-01],
        [ 3.6770e-01],
        [ 3.6535e-03],
        [ 4.2364e-01],
        [-5.4703e-01],
        [ 8.9173e-02],
        [-3.9032e-01],
        [-5.9740e-01],
        [ 3.7479e-02],
        [ 3.0257e-01],
        [ 8.2539e-02],
        [-6.0559e-01],
        [-4.3660e-01],
        [-7.0624e-01],
        [-5.0503e-01],
        [-4.0929e-01],
        [-2.3300e-01],
        [ 2.0298e-01],
        [-6.3697e-01],
        [-1.2584e-01],
        [ 5.6092e-02],
        [ 5.0150e-02],
        [-1.5358e-01],
        [ 2.9248e-02],
        [ 1.1180e-01],
        [-1.5535e-01],
        [ 1.1964e-01],
        [-6.5698e-01],
        [ 4.1923e-01],
        [ 7.4044e-02],
        [ 2.4536e-02],
        [ 3.2647e-01],
        [-7.7464e-01],
        [ 3.9898e-01],
        [-2.5777e-01],
        [ 8.5569e-02],
        [-4.0305e-01],
        [ 5.4463e-01],
        [-3.4124e-01],
        [-4.0789e-01],
        [ 4.2093e-01],
        [-3.8487e-01],
        [-4.0491e-01],
        [-2.1539e-01],
        [-1.7979e-02],
        [ 3.2492e-01],
        [-2.0894e-01],
        [ 2.5629e-01],
        [ 9.6046e-01]], device='cuda:0', grad_fn=<AddmmBackward0>)

Output:
That tensor has a singleton dimension (ie. it's of shape [Nx1]). Just squeeze that dimension or pick the 0th element:
In [1]: import torch

In [2]: a = torch.zeros([10,1])

In [3]: a
Out[3]:
tensor([[0.],
        [0.],
        [0.],
        [0.],
        [0.],
        [0.],
        [0.],
        [0.],
        [0.],
        [0.]])

In [4]: a[:,0]
Out[4]: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])

In [5]: a.squeeze(1)
Out[5]: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])


Instruction: is there a faster way to compare two raws in a two tensors than for loop?

Input:
im new to pytorch and i would like to check for each element in the tensor1 if it is part of the same row in the tensor2 for example if i have two tensor as follwoing:
a = torch.tensor([[0,5],[1,5], [4,5], [7,8]])
b = torch.tensor([[0,1],[2,3], [7,5], [-1,7]])

the output should be as following:
[ True, False]
[False, False]
[False, False]
[False, False]
[False, False]
[False,  True]
[False,  True]
[False, False]

i know that i can do it using a simple for loop but is there a way to do it faster in pytorch?

Output:
you need to use the repeat_interleave for both tensors and then reshape the first tensor, and then you compare it with the other tensor
like this
res = a.repeat_interleave(2, dim=1).reshape(-1, 2) == b.repeat_interleave(2, dim=0)

the output should be as follows
output
tensor([[ True, False],
        [False, False],
        [False, False],
        [False, False],
        [False, False],
        [False,  True],
        [False,  True],
        [False, False]])

the output is tensor. If you need it without [] you can use the flatten method


Instruction: How to initialise (and check sanity) weights efficiently of layers within complex (nested) modules in PyTorch?

Input:
Looking for an efficient way to access nested Modules and Layers to set the weights
I am replicating the DCGAN Paper and my code works as expected. I found out that in the paper, the authors said that:

All weights were initialized from a zero-centered Normal distribution
with standard deviation 0.02

This awesome answer explains that it can be done using torch.nn.init.normal_(nn.Conv2d(1,1,1, 1,1 ).weight.data, 0.0, 0.02) but I have complex structure using ModuleList and others. What is the most efficient way of doing this?
By Complex, please look at the code below for my implementation:
'''
Implement the Deep Convolution Gan AKA DCGAN in Pytorch: Paper at https://arxiv.org/pdf/1511.06434v2.pdf
'''
import torch
import torch.nn as nn

class GeneratorBlock(nn.Module):
    '''
    Generator Block uses TransposedConv2D -> Batch Norm (except LAST block) -> Relu
    Note: kernel_size = 4, stride = 2, padding = 1 is used in the paper. When BatchNorm is used, Bias is not used for Conv2D
    '''
    def __init__(self, in_channels, out_channels, kernel_size = 4, stride = 2, padding = 1, use_batchnorm:bool = True):
        super().__init__()
        self.use_batchnorm = use_batchnorm
        self.transpose_conv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size = kernel_size, stride=stride, padding=padding, bias = not self.use_batchnorm)
        self.batch_norm = nn.BatchNorm2d(out_channels) if self.use_batchnorm else None
        self.activation = nn.ReLU() # Paper uses Relu in Generator Network

    def forward(self, x):
        x = self.transpose_conv(x)
        return self.activation(self.batch_norm(x)) if self.use_batchnorm else self.activation(x)

class Generator(nn.Module):
    '''
    Generate Images using Transposed Convolution. Input is a random noise of [Batch, 100, 1,1] Dimension and then upsampled
    '''
    def __init__(self, input_features = 100, base_feature = 128, final_channels:int = 1):
        '''
        We use nn.Sequantial here just to show the workings. If you want to make the layers dynamically using a loop, find nn.ModuleList() in the Descriminator block. Both works same
        So we'll use 'base_feature = 64' as a base for input and output channels
        args:
            input_features: The shape of Random Noise from which an image will be generated
            base_feature: The shape of feature map or number or channels which will act as out base. Other inputs and outputs will be calculated based on this
            final_channels: The channels / features which will be sent to the Discriminator as an input
        '''
        super(Generator, self).__init__()

        # in Descriminator, we do the same work using ModuleList(). Uses 4 blocks
        self.blocks = nn.Sequential(
            GeneratorBlock(in_channels = input_features, out_channels = base_feature * 8, stride = 1, padding = 0), # from Random Noise, Generate 1024 features
            GeneratorBlock(in_channels = base_feature * 8, out_channels = base_feature * 4), # 1024 -> 512 features
            GeneratorBlock(in_channels = base_feature * 4, out_channels = base_feature * 2), # 512 -> 256 features
            GeneratorBlock(in_channels = base_feature * 2, out_channels = base_feature), # 256 -> 128 features
            nn.ConvTranspose2d(base_feature, final_channels, kernel_size = 4, stride = 2, padding = 1)# 128 -> final feature. It is just GeneratorBlock without ReLu and BatchNorm ;)
        )
        self.activation = nn.Tanh() # To make the outputs between [-1,1]

    def forward(self, x):
        '''
        Takes Random Noise as input and Generte features from that
        '''
        return self.activation(self.blocks(x))

class DiscriminatorBlock(nn.Module):
    '''
    Discriminator Block uses Conv2D -> Batch Norm (except FIRST block) -> LeakyRelu
    Note: kernel_size = 4, stride = 2, padding = 1 is used in the paper. When BatchNorm is used, Bias is not used for Conv2D
    '''
    def __init__(self, in_channels, out_channels, kernel_size = 4, stride = 2, padding = 1, use_batchnorm:bool = True):
        super().__init__()
        self.use_batchnorm = use_batchnorm
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias = not self.use_batchnorm)
        self.batch_norm = nn.BatchNorm2d(out_channels) if self.use_batchnorm else None
        self.activation = nn.LeakyReLU(0.2)

    def forward(self, x):
        x = self.conv(x)
        return self.activation(self.batch_norm(x)) if self.use_batchnorm else self.activation(x)

class Discriminator(nn.Module):
    '''
    CNNs to classify whether the image generated by the Generator are as good as the real ones
    Feature Changes as :: 1 -> 64 -> 128 -> 256 -> 512 -> 1
    '''
    def __init__(self, input_features = 1, output_features = 1,  middle_features = [64,128,256]):
        '''
        In the paper, they take in a feature of [Batch, 1, 64, 64] from the Generator and then output a single number per sample in the batch
        '''
        super().__init__()
        self.layers = nn.ModuleList() # Just a fancy method of stacking layers using loop

        # in the paper, the first layer does not use BatchNorm
        self.layers.append(DiscriminatorBlock(input_features, middle_features[0], use_batchnorm = False)) #  1 -> 64 Because the input has 1 channel

        for i, channel in enumerate(middle_features): # total 4 blocks are used in paper. 1 has already been used in the line above. 3 blocks are these
            self.layers.append(DiscriminatorBlock(channel, channel*2)) # 64 -> 128 --- 128 -> 256 --- 256 -> 512

        self.final_conv = nn.Conv2d(in_channels = middle_features[-1]*2,  out_channels = output_features, kernel_size = 4, stride = 2,  padding = 0) # Input from previous layer 512 -> 1
        self.sigmoid_layer = nn.Sigmoid() # gives whether an image is real or fake or more precisely, how CLOSE is it to the real image

    def forward(self, x):
        for layer in self.layers:
            x = layer(x)

        return self.sigmoid_layer(self.final_conv(x))

def test_DCGAN_code():
    noise = torch.rand(10,100,1,1)
    image = Generator()(noise)
    result = Discriminator()(image)
    print('Model Built Successfully!!! Generating 10 random samples and their end results')
    print(f"'Z' random Noise shape: {noise.shape} || Generator output shape: {image.shape} || Discriminator shape: {result.shape}")

Output:
You can simply iterate over all submodules, at the end of your __init__ method:
class Generator(nn.Module):
  def __init__(self, ....):
    # all code here
    # ...
    # init weights, at the very bottom of __init__
   for sm in self.modules():
     if isinstance(sm, nn.Conv2d):
       # only conv2d will be initialized in this way
       torch.nn.init.normal_(sm.weight.data, 0.0, 0.02)

done.


Instruction: PyTorch moving average computation creates inplace operation

Input:
I have a loss function that depends on an "exponential moving average" Z. A minimal example (pay special attention to the getUpdatedZ function):
import torch
import torch.nn as nn

class FeedForward(nn.Module):
    def __init__(self):
        super(FeedForward, self).__init__()

        self.model = nn.Sequential(nn.Linear(1, 100),
                                   nn.ReLU(),
                                   nn.Linear(100, 1))

    def forward(self, x):
        return self.model(x)

model = FeedForward()
nEpochs = 100

optimizer = torch.optim.Adam(params=model.parameters(), lr=1e-3)

def getTrainingPoints():
  return torch.rand(1000, 1)

def lossFunction(X, Z):
  # Returning Z here is enough to expose the problem. The real loss is more complicated.
  return Z

def getUpdatedZ(X, Z):
  U = model(X)
  Znew = torch.mean(U)
  # Having Z in this computation creates an inplace operation (I'm not sure why).
  # Returning, for example, Znew, does not cause any issues (but the computation is incorrect)
  return 0.2 * Z + 0.8 * Znew

Z = torch.tensor([1.0])
X = getTrainingPoints()
for i in range(nEpochs):
  optimizer.zero_grad()
  Z = getUpdatedZ(X, Z)
  loss = lossFunction(X, Z)
  # loss function depends on gradient of the model in the real version of the code, hence retain_graph=True
  loss.backward(retain_graph=True)
  optimizer.step()

I get the following error:
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [100, 1]], which is output 0 of AsStridedBackward0, is at version 2; expected version 1 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

Output:
After some trials, I think that the error arises because you are computing a recursive function (Z = getUpdatedZ(X, Z)) but you are changing some of its parameters (the weights of the Linear modules) at each iteration through optimizer.step().
You can backward() just at the end of the for cycle, or you may want to break the autodifferentiation graph, for example by calling Z.detach() after loss.backward(). Sometimes this trick is used to avoid too complex and inefficient backpropagations (check, for example this).
However in both cases, this will change the structure of the optimized function, so be sure of what you are doing.


Instruction: Calculating dimensions of fully connected layer?

Input:
I am struggling to work out how to calculate the dimensions for the fully connected layer. I am inputing images which are (448x448) using a batch size (16). Below is the code for my convolutional layers:
class ConvolutionalNet(nn.Module):
  def __init__(self, num_classes=182):
    super().__init__()

    self.layer1 = nn.Sequential(
        nn.Conv2d(3, 16, kernal_size=5, stride=1, padding=2),
        nn.BatchNorm2d(16),
        nn.ReLU(),
        nn.MaxPool2d(kernal_size=2, stride=2)
    )

    self.layer2 = nn.Sequential(
        nn.Conv2d(16, 32, kernal_size=5, stride=1, padding=2),
        nn.BatchNorm2d(32),
        nn.ReLU(),
        nn.MaxPool2d(kernal_size=2, stride=2)
    )

    self.layer3 = nn.Sequential(
        nn.Conv2d(32, 32, kernal_size=5, stride=1, padding=2),
        nn.BatchNorm2d(32),
        nn.ReLU(),
        nn.MaxPool2d(kernal_size=2, stride=2)
    )

    self.layer4 = nn.Sequential(
        nn.Conv2d(32, 64, kernal_size=5, stride=1, padding=2),
        nn.BatchNorm2d(64),
        nn.ReLU(),
        nn.MaxPool2d(kernal_size=2, stride=2)
    )

    self.layer5 = nn.Sequential(
        nn.Conv2d(64, 64, kernal_size=5, stride=1, padding=2),
        nn.BatchNorm2d(64),
        nn.ReLU(),
        nn.MaxPool2d(kernal_size=2, stride=2)
    )

I want to add a fully connected layer:
self.fc = nn.Linear(?, num_classes)

Would anyone be able to explain the best way to go about calculating this? Also, if I have multiple fully connected layers e.g. (self.fc2, self.fc3), would the second parameter always equal the number of classes. I am new to coding and finding it hard to wrap my head around this.

Output:
The conv layers don't change the width/height of the features since you've set padding equal to (kernel_size - 1) / 2. Max pooling with kernel_size = stride = 2 will decrease the width/height by a factor of 2 (rounded down if input shape is not even).
Using 448 as input width/height, the output width/height will be 448 // 2 // 2 // 2 // 2 // 2 = 448/32 = 14 (where // is floor-divide operator).
The number of channels is fully determined by the last conv layer, which outputs 64 channels.
Therefore you will have a [B,64,14,14] shaped tensor, so the Linear layer should have in_features = 64*14*14 = 12544.
Note you'll need to flatten the input beforehand, something like.
self.layer6 = nn.Sequential(
    nn.Flatten(),
    nn.Linear(12544, num_classes)
)


Instruction: RuntimeError: Given groups=1, weight of size [32, 16, 5, 5], expected input[16, 3, 448, 448] to have 16 channels, but got 3 channels instead

Input:
I am getting the following error and can't figure out why. I printed the input size of my torch before it gets fed to the CNN:
torch.Size([16, 3, 448, 448])

Here is my error message:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-116-bfa18f2a99fd> in <module>()
     14     # Forward pass
     15     print(images.shape)
---> 16     outputs = modelll(images.float())
     17     loss = criterion(outputs, labels)
     18

6 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias)
    441                             _pair(0), self.dilation, self.groups)
    442         return F.conv2d(input, weight, bias, self.stride,
--> 443                         self.padding, self.dilation, self.groups)
    444
    445     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Given groups=1, weight of size [32, 16, 5, 5], expected input[16, 3, 448, 448] to have 16 channels, but got 3 channels instead

I defined a CNN with 5 convolutional layers and two fully connected layers. I am feeding in batches of 16 and have resized the images to be (448x448). The images are colour, so I assumed an input of torch.Size([16, 3, 448, 448]) would be correct. Do I need to rearrange my tensor to be torch.Size([3, 448, 448, 16])? Just guessing here as I am fairly new to coding. I have looked online but haven't been able to figure it out. Any help would be greatly appreciated.
#Defining CNN
class ConvolutionalNet(nn.Module):
  def __init__(self, num_classes=182):
    super().__init__()

    self.layer1 = nn.Sequential(
        nn.Conv2d(3, 16, kernel_size=5, stride=1, padding=2),
        nn.BatchNorm2d(16),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2)
    )

    self.layer2 = nn.Sequential(
        nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),
        nn.BatchNorm2d(32),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2)
    )

    self.layer3 = nn.Sequential(
        nn.Conv2d(32, 32, kernel_size=5, stride=1, padding=2),
        nn.BatchNorm2d(32),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2)
    )

    self.layer4 = nn.Sequential(
        nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=2),
        nn.BatchNorm2d(64),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2)
    )

    self.layer5 = nn.Sequential(
        nn.Conv2d(64, 64, kernel_size=5, stride=1, padding=2),
        nn.BatchNorm2d(64),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2)
    )

    self.fc1 = nn.Linear(10*10*64, 240)
    self.fc2 = nn.Linear(240, 182)

  def forward(self, x):
    out = self.layer1(x)
    out = self.layer2(x)
    out = self.layer3(x)
    out = self.layer4(x)
    out = self.layer5(x)
    out = out.reshape(out.size(0), -1)
    out = F.relu(self.fc1((x)))
    out = self.fc3(x)

    return out

#Creating instance
modelll = ConvolutionalNet(num_classes).to(device)
modelll

num_classes = 182

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(modelll.parameters(), lr=3e-4)

#Training loop
modelll.train()
num_epochs = 5    # Set the model into `training` mode, because certain operators will perform differently during training and evaluation (e.g. dropout and batch normalization)
total_epochs = notebook.tqdm(range(num_epochs))

for epoch in total_epochs:
  for i, (images, labels, m) in enumerate(train_loader):
    # Move tensors to the configured device
    images = images.to(device)
    labels = labels.to(device)
    # Forward pass
    print(images.shape)
    outputs = modelll(images.float())
    loss = criterion(outputs, labels)

    # Backward and optimize
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    if (i + 1) % 10 == 0:
      total_epochs.set_description(
          'Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(
              epoch + 1, num_epochs, i + 1, len(train_loader), loss.item()))

Output:
You haven't passed your output to the next layer's input, you're continually using the input. You should change your forward call to:
  def forward(self, x):
    out = self.layer1(x)
    out = self.layer2(out)
    out = self.layer3(out)
    out = self.layer4(out)
    out = self.layer5(out)
    out = out.reshape(out.size(0), -1)
    out = F.relu(self.fc1((out)))
    out = self.fc3(out)
    return out


Instruction: Pytorch Custom dataloader: TypeError: pic should be PIL Image or ndarray. Got

Input:
I want to use a custom data loader to transfer numpy files to a dataloader. When I set the transorm then I get the error  TypeError: pic should be PIL Image or ndarray. Got <class 'torch.Tensor'>
import os
import torch
import numpy as np
from torch.utils.data import Dataset, TensorDataset, DataLoader
from torchvision import transforms

class CustomTensorDataset(Dataset):
    """
    TensorDataset with support for transforms
    """
    def __init__(self, tensors, transform=None):
        assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        self.transform = transform

    def __getitem__(self, index):
        x = self.tensors[0][index]

        if self.transform:
            x = self.transform(x)

        y = self.tensors[1][index]

        return x, y

    def __len__(self):
        return self.tensors[0].size(0)

te_data    =  torch.FloatTensor(np.ones([100, 3, 32, 32]))
te_targets =  torch.FloatTensor(np.ones([100]))

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010])
])

testset_custom = CustomTensorDataset(tensors=[te_data, te_targets], transform=transform)
# testset_custom = CustomTensorDataset(tensors=[te_data, te_targets], transform=None) # --> no error

for item in testset_custom:
    print(item)

Output:
Your input data to the Dataset need to be PIL image or numpy array. However, your te_data and te_targets are torch.tensor. To solve this just do not convert them to torch.tensor and before giving to Dataset and keep their dimension. Dataset, itself changes its dimension:
te_data    =  np.ones([100, 32, 32, 3])
te_targets =  np.ones([100])

And also the condition with assert need to be changed as far as the input is numpy array:
assert all(tensors[0].shape[0] == tensor.shape[0] for tensor in tensors)


Instruction: Pytorch datasets.UDPOS.splits throwing error

Input:
I want to split the UDPOS dataset into train, valid, and test by fields. Below is my code-
import torch
import torch.nn as nn
import torch.optim as optim

from torchtext.legacy import data
from torchtext import datasets

SEED = 1234

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True

TEXT = data.Field(lower = True)
UD_TAGS = data.Field(unk_token = None)
PTB_TAGS = data.Field(unk_token = None)

fields = (("text", TEXT), ("udtags", UD_TAGS), ("ptbtags", PTB_TAGS))

train_data, valid_data, test_data = datasets.UDPOS.splits(fields)

This code give me following error-

I am using Pytorch version - '1.10.2'.
How do I split the UDPOS dataset using fileds in the current version.

Output:
I solved the same problem by changing the code
from torchtext import datasets

to
from torchtext.legacy import datasets


Instruction: How to access latest torchvision.models (e.g. ViT)?

Input:
I have seen in the official torchvision docs that recently vision transformers and the ConvNeXt model families have been added to the PyTorch model zoo. However, even after upgrading to latest torchvision version 0.11.3 (via pip) these new models are not available:
>>> import torchvision; torchvision.__version__
'0.11.3+cu102'
>>> import torchvision.models as models
>>> model = models.resnext50_32x4d()  # previous models work fine
>>> model = models.vit_b_16()  # vision transformers don't work
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: module 'torchvision.models' has no attribute 'vit_b_16'

Any ideas how I can access these latest model additions in PyTorch?

Output: Event though @Shai's answer is a nice addition, my original question was how I could access the official ViT and ConvNeXt models in torchvision.models. As it turned out the answer was simply to wait. So for the records: After upgrading to latest torchvision pip package in version 0.12 I got these new models as well.


Instruction: Pytorch gradient error: nonetype unsupported operand type(s) for +: 'NoneType' and 'NoneType'

Input:
I get the following error message when i run the code given below. The code is a part of a bigger code but essentially i am computing gradients here. The variable displacements and colloc_points are of the type: tensor([[-0.0819,  0.1623,  0.1228]], grad_fn=),
tensor([[-0.0556,  2.2222,  0.1667]], requires_grad=True) respectively. How do i resolve this error?

---> 43   eps_xy = 0.5*(u_y+v_x)

TypeError: unsupported operand type(s) for +: 'NoneType' and 'NoneType'

  u = displacement[0,0]
  v = displacement[0,1]
  w = displacement[0,2]

  print(colloc_point)

  x = colloc_point[0,0]
  y = colloc_point[0,1]
  z = colloc_point[0,2]

  ## Compute gradients

  u_y = torch.autograd.grad(u, y, create_graph=True,allow_unused=True)[0]
  v_x = torch.autograd.grad(v, x, create_graph=True,allow_unused=True)[0]

eps_xy = 0.5*(u_y+v_x)```

Output:
It is possible that, even if u, y, v and x require the gradient, u is not a function of y and, similarly v does not depend on x.
You can check this by setting allow_unused=False and see if torch.autograd.grad returns an error.


Instruction: Semantic segmentation with detectron2

Input:
I used Detectron2 to train a custom model with Instance Segmentation and worked well. There are several Tutorials on google colab with Detectron2 using Instance Segmentation, but nothing about Semantic Segmentation. So, to train the Custom Instance Segmentation the code based on colab (https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5#scrollTo=7unkuuiqLdqd) is this:
from detectron2.engine import DefaultTrainer

cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.DATASETS.TRAIN = ("balloon_train",)
cfg.DATASETS.TEST = ()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")  # Let training initialize from model zoo
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = 0.00025  # pick a good LR
cfg.SOLVER.MAX_ITER = 300    # 300 iterations seems good enough for this toy dataset; you will need to train longer for a practical dataset
cfg.SOLVER.STEPS = []        # do not decay learning rate
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128   # faster, and good enough for this toy dataset (default: 512)
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  # only has one class (ballon). (see https://detectron2.readthedocs.io/tutorials/datasets.html#update-the-config-for-new-datasets)
# NOTE: this config means the number of classes, but a few popular unofficial tutorials incorrect uses num_classes+1 here.

os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

And to run Semantic Segmentation train I replaced "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml" to "/Misc/semantic_R_50_FPN_1x.yaml", basicly I changed the pre-trainded model, just this. And I got this error:
TypeError: cross_entropy_loss(): argument 'target' (position 2) must be Tensor, not NoneType

How I set up to Semantic Segmentation on Google Colab?

Output:
To train for semantic segmentation you can use the same COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml model. You don't have to change this line.
The training code you showed in your question is correct and can be used for semantic segmentation as well. All that changes are the label files.
Once the model is trained, you can use it for inference by loading the model weights from the trained model
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set the testing threshold for this model
cfg.DATASETS.TEST = ("Detectron_terfspot_" + "test", )                      # the name given to your dataset when loading/registering it
cfg.DATALOADER.NUM_WORKERS = 2
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
predictor = DefaultPredictor(cfg)


Instruction: How can I create images for each batch using Pytorch?

Input:
I want to make a binary classifier that classifies the following:
Class 1. Some images that I already have.
Class 2. Some images that I create from a function, using the images of class 1.
The problem is that instead of pre-creating the two classes, and then loading them, to speed up the process I would like the class 2 images to be created for each batch.
Any ideas on how I can tackle the problem? If I use the DataLoader as usual, I have to enter the images of both classes directly, but if I still don't have the images of the second class I don't know how to do it.
Thanks.

Output:
You can tackle the problem in at least two ways.

(Preferred) You create a custom Dataset class, AugDset, such that AugDset.__len__() returns 2 * len(real_dset), and when idx > len(imgset), AugDset.__getitem__(idx) generates the synthetic image from real_dset(idx).
You create your custom collate_fn function, to be passed to DataLoader that, given a batch, it augments it with your synthetic generated images.


Instruction: RuntimeError: 0D or 1D target tensor expected, multi-target not supported I was training a deep learning model but i am getting this issue

Input:
*My Training Model*
def train(model,criterion,optimizer,iters):
    epoch = iters
    train_loss = []
    validaion_loss = []
    train_acc = []
    validation_acc = []
    states = ['Train','Valid']
    for epoch in range(epochs):
        print("epoch : {}/{}".format(epoch+1,epochs))
        for phase in states:
            if phase == 'Train':
                model.train() *training the data if phase is train*
                dataload = train_data_loader
            else:
                model.eval()
                dataload = valid_data_loader

            run_loss,run_acc = 0,0 *creating variables to calculate loss and acc*
            for data in dataload:
                inputs,labels = data
                inputs = inputs.to(device)
                labels = labels.to(device)

                labels = labels.byte()
                optimizer.zero_grad() #Using the optimizer

                with torch.set_grad_enabled(phase == 'Train'):
                    outputs = model(inputs)
                    loss = criterion(outputs,labels.unsqueeze(1).float())

                    predict = outputs>=0.5
                    if phase == 'Train':
                        loss.backward() #backward propagation
                        optimizer.step()

                    acc = torch.sum(predict == labels.unsqueeze(1))
                run_loss+=loss.item()
                run_acc+=acc.item()/len(labels)
            if phase == 'Train': #calulating train loss and accucracy
                epoch_loss = run_loss/len(train_data_loader)
                train_loss.append(epoch_loss)
                epoch_acc = run_acc/len(train_data_loader)
                train_acc.append(epoch_acc)
            else: #training validation loss and accuracy
                epoch_loss = run_loss/len(valid_data_loader)
                validaion_loss.append(epoch_loss)
                epoch_acc = run_acc/len(valid_data_loader)
                validation_acc.append(epoch_acc)

            print("{}, loss :{},accuracy:{}".format(phase,epoch_loss,epoch_acc))

    history = {'Train_loss':train_loss,'Train_accuracy':train_acc,
               'Validation_loss':validaion_loss,'Validation_Accuracy':validation_acc}
    return model,history[enter image description here][1]

I was experiencing the error as 0D or 1D target tensor expected, multi-target not supported could you please help in rectifying the code which is described above. Referred the previous related articles but unable to get the desired result. What are the code snippets I had to change so that my model will run successfully. Any suggestions are mostly welcome. Thanks in Advance.

Output:
Your problem is that labels have the correct shape to calculate the loss. When you add .unsqueeze(1) to labels you made your labels with this shape [32,1] which is not consistent to the requirment to calcualte the loss.
To fix the problem, you only need to remove .unsqueeze(1) for labels.
If you read the documentation of CrossEntropLoss, the arguments:

Input should be in (N,C) shape which is outputs in your case and [32,3].
Target should be in N shape which is labels in your case and should be [32]. Therefore, the loss function expects labels to be in 1D target not multi-target.


Instruction: Hugginface Dataloader BERT ValueError: too many values to unpack (expected 2), AX hyperparameters tuning with Pytorch

Input:
I'm stacked with this error from one week now, I tried everything so the fact is that I'm not understanding deeply what is happening (I'm new at pytorch implementation). Anyway I'm trying to implement a Bert Classifier to discriminate between 2 sequences classes, with AX hyperparameters tuning.
This is all my code implemented anticipated by a sample of my datasets ( I have 3 csv, train-test-val). Thank you very much !
                                           0        1
M A T T D R P T P D G T D A I D L T T R V R R...    1
M K K L F Q T E P L L E L F N C N E L R I I G...    0
M L V A A A V C P H P P L L I P E L A A G A A...    1
M I V A W G N S G S G L L I L I L S L A V S A...    0
M V E E G R R L A A L H P N I V V K L P T T E...    1
M G S K V S K N A L V F N V L Q A L R E G L T...    1
M P S K E T S P A E R M A R D E Y Y M R L A M...    1
M V K E Y A L E W I D G Y R E R L V K V S D A...    1
M G T A A S Q D R A A M A E A A Q R V G D S F...    0

df_train=pd.read_csv('CLASSIFIER_train',sep=',',header=None)
df_train
class SequenceDataset(Dataset):

  def __init__(self, sequences, targets, tokenizer, max_len):
    self.sequences = sequences
    self.targets = targets
    self.tokenizer = tokenizer
    self.max_len = max_len

  def __len__(self):
    return len(self.sequences)

  def __getitem__(self, item):
    sequences = str(self.sequences[item])
    target = self.targets[item]

    encoding = self.tokenizer.encode_plus(
      sequences,
      add_special_tokens=True,
      max_length=self.max_len,
      return_token_type_ids=False,
      pad_to_max_length=True,
      return_attention_mask=True,
      return_tensors='pt',
    )

    return {
      'sequences_text': sequences,
      'input_ids': encoding['input_ids'].flatten(),
      'attention_mask': encoding['attention_mask'].flatten(),
      'targets': torch.tensor(target, dtype=torch.long)
    }

def create_data_loader(df, tokenizer, max_len, batch_size):
  ds = SequenceDataset(
    sequences=df[0].to_numpy(),
    targets=df[1].to_numpy(),
    tokenizer=tokenizer,
    max_len=max_len
  )

  return DataLoader(
    ds,
    batch_size=batch_size,
    num_workers=2,
    shuffle=True
  )

BATCH_SIZE = 16

train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)

def net_train(net, train_data_loader, parameters, dtype, device):
  net.to(dtype=dtype, device=device)

  # Define loss and optimizer
  criterion = nn.CrossEntropyLoss()
  optimizer = optim.SGD(net.parameters(), # or any optimizer you prefer
                        lr=parameters.get("lr", 0.001), # 0.001 is used if no lr is specified
                        momentum=parameters.get("momentum", 0.9)
  )

  scheduler = optim.lr_scheduler.StepLR(
      optimizer,
      step_size=int(parameters.get("step_size", 30)),
      gamma=parameters.get("gamma", 1.0),  # default is no learning rate decay
  )

  num_epochs = parameters.get("num_epochs", 3) # Play around with epoch number
  # Train Network
  for _ in range(num_epochs):
      for inputs, labels in train_data_loader:
          # move data to proper dtype and device
          inputs = inputs.to(dtype=dtype, device=device)
          labels = labels.to(device=device)

          # zero the parameter gradients
          optimizer.zero_grad()

          # forward + backward + optimize
          outputs = net(inputs)
          loss = criterion(outputs, labels)
          loss.backward()
          optimizer.step()
          scheduler.step()
  return net

def init_net(parameterization):

    model = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)

    # The depth of unfreezing is also a hyperparameter
    for param in model.parameters():
        param.requires_grad = False # Freeze feature extractor

    Hs = 512 # Hidden layer size; you can optimize this as well

    model.fc = nn.Sequential(nn.Linear(2048, Hs), # attach trainable classifier
                                 nn.ReLU(),
                                 nn.Dropout(0.2),
                                 nn.Linear(Hs, 10),
                                 nn.LogSoftmax(dim=1))
    return model # return untrained model

def train_evaluate(parameterization):

    # constructing a new training data loader allows us to tune the batch size

    train_data_loader=create_data_loader(df_train, tokenizer, MAX_LEN, batch_size=parameterization.get("batchsize", 32))

    # Get neural net
    untrained_net = init_net(parameterization)

    # train
    trained_net = net_train(net=untrained_net, train_data_loader=train_data_loader,
                            parameters=parameterization, dtype=dtype, device=device)

    # return the accuracy of the model as it was trained in this run
    return evaluate(
        net=trained_net,
        data_loader=test_data_loader,
        dtype=dtype,
        device=device,
    )

classes=('0','1')

dtype = torch.float
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

best_parameters, values, experiment, model = optimize(
    parameters=[
        {"name": "lr", "type": "range", "bounds": [1e-6, 0.4], "log_scale": True},
        {"name": "batchsize", "type": "range", "bounds": [16, 128]},
        {"name": "momentum", "type": "range", "bounds": [0.0, 1.0]},
        #{"name": "max_epoch", "type": "range", "bounds": [1, 30]},
        #{"name": "stepsize", "type": "range", "bounds": [20, 40]},
    ],

    evaluation_function=train_evaluate,
    objective_name='accuracy',
)

print(best_parameters)
means, covariances = values
print(means)
print(covariances)

  File "<ipython-input-71-e52ebc0d7b5b>", line 14, in train_evaluate
    parameters=parameterization, dtype=dtype, device=device)
  File "<ipython-input-61-66c57e7138fa>", line 20, in net_train
    for inputs, labels in train_data_loader:
ValueError: too many values to unpack (expected 2)

Output:
your dataloader returns a dictionary therefore the way you loop and access it is wrong should be done as such:
# Train Network
  for _ in range(num_epochs):
      # Your dataloader returns a dictionary
      # so access it as such
      for batch in train_data_loader:
          # move data to proper dtype and device
          labels = batch['targets'].to(device=device)
          atten_mask = batch['attention_mask'].to(device=device)
          input_ids = batch['input_ids'].to(device=device)

          # zero the parameter gradients
          optimizer.zero_grad()

          # forward + backward + optimize
          outputs = net(input_ids, attention_mask=atten_mask)


Instruction: Using more than 1 metric in pytorch

Input:
I have some experience in Tenserflow but I'm new to pytorch. Sometimes I need more than 1 metric to check the accuracy of training. In Tenserflow, I used to do as shown below. But I wonder how could list more than 1 metric in pytorch.
LR = 0.0001
optim = keras.optimizers.Adam(LR)

dice_loss_se2 = sm.losses.DiceLoss()
mae = tf.keras.losses.MeanAbsoluteError( )
metrics = [ mae,sm.metrics.IOUScore(threshold=0.5), sm.metrics.FScore(threshold=0.5) , dice_loss_se2]

model.compile(optimizer=optim,loss= dice_loss_se2,metrics= metrics)

Output:
In pytorch training is done mostly through loops so you have define via each step, there are packages like torchmetrics which you can run each metric heres an example:
import torchmetrics

for step, (test_image, test_labels) in tqdm(enumerate(test_dataloader), total=len(test_dataloader)):
        test_batch_image = test_image.to('cuda')
        test_batch_label = test_labels.to('cuda')
        targets.append(test_labels)

        with torch.no_grad():
            logits = model(test_batch_image)

        loss = criterion(logits, test_batch_label)
        test_loss += loss.item()

        preds.append(logits.detach().cpu().numpy().argmax(axis=1))

    preds = torch.tensor(np.concatenate(preds))
    targets = torch.tensor(np.concatenate(targets))
    print('[Epoch %d] Test loss: %.3f' %(epoch + 1, test_loss/ len(test_dataloader)))
    print('Accuracy: {}%'.format(round(torchmetrics.functional.accuracy(target=targets, preds=preds).item() * 100), 2))


Instruction: Why does torch transpose return a non-contiguous tensor but the equivalent view operation returns a contiguous one?

Input:
Both torch.Tensor.view() and torch.tensor.transpose() can transpose a 2 dimensional tensor, e.g.
a = torch.arange(8).reshape(2, 4)
a.t().is_contiguous() # False
a.view(4,2).is_contiguous() # True

But exchanging the dimensions with .view() results in a contiguous tensor while using .t() results in a non-contiguous one. It's clearly somehow possible to transpose while retaining contiguity, though I don't really understand how, but my main question is: why is this behavior different for these two functions?
The documentation for view says that the "returned tensor shares the same data" and for transpose that the "resulting out tensor shares its underlying storage with the input tensor" so I naively expect the contiguity status of the outputs to be the same for both functions, but perhaps "sharing the same data" and "sharing underlying storage" are two different things?
Edit: It did not occur to me that there is a way to reshape this tensor into 4,2 that is not tansposing lol. Thanks to the author of the accepted answer for pointing out that a.t() != a.view(dim1, dim0).

Output:
With view(4, 2) you are not transposing the tensor, you are simply reshaping it.
Original tensor:
t = torch.tensor([[1, 2],
                  [3, 4],
                  [5, 6],
                  [7, 8]])

t.transpose() returns:
torch.tensor([[1, 3, 5, 7],
              [2, 4, 6, 8]])

t.view(4, 2) returns:
t.tensor([[1, 2, 3, 4],
          [5, 6, 7, 8]])


Instruction: Training with threshold in PyTorch

Input:
I have a neural network, which produces a single value when excited with input. I need to use this value returned by the network to threshold another array. The result of this threshold operation is used to compute a loss function (the value of threshold is not known before hand and needs to be arrived at by training).
Following is an MWE
import torch

x = torch.randn(10, 1)  # Say this is the output of the network (10 is my batch size)
data_array = torch.randn(10, 2)  # This is the data I need to threshold
ground_truth = torch.randn(10, 2)  # This is the ground truth
mse_loss = torch.nn.MSELoss()  # Loss function

# Threshold
thresholded_vals = data_array * (data_array >= x)  # Returns zero in all places where the value is less than the threshold, the value itself otherwise

# Compute loss and gradients
loss = mse_loss(thresholded_vals, ground_truth)
loss.backward()  # Throws error here

Since the operation of thresholding returns a tensor array that is devoid of any gradients the backward() operation throws error.
How does one train a network in such a case?

Output:
Your threshold function is not differentiable in the threshold, therefore torch does not calculate the gradient for the threshold which is why your example is not working.
import torch

x = torch.randn(10, 1, requires_grad=True)  # Say this is the output of the network (10 is my batch size)
data_array = torch.randn(10, 2, requires_grad=True)  # This is the data I need to threshold
ground_truth = torch.randn(10, 2)  # This is the ground truth
mse_loss = torch.nn.MSELoss()  # Loss function

# Threshold
thresholded_vals = data_array * (data_array >= x)  # Returns zero in all places where the value is less than the threshold, the value itself otherwise

# Compute loss and gradients
loss = mse_loss(thresholded_vals, ground_truth)
loss.backward()  # Throws error here
print(x.grad)
print(data_array.grad)

Output:
None #<- for the threshold x
tensor([[ 0.1088, -0.0617],  #<- for the data_array
        [ 0.1011,  0.0000],
        [ 0.0000,  0.0000],
        [-0.0000, -0.0000],
        [ 0.2047,  0.0973],
        [-0.0000,  0.2197],
        [-0.0000,  0.0929],
        [ 0.1106,  0.2579],
        [ 0.0743,  0.0880],
        [ 0.0000,  0.1112]])


Instruction: How to add an additional output node during training for Pytorch?

Input:
I am making a class-incremental learning multi-label classifier. Here the model first trains with 7 labels. After training, another dataset emerges that contains the same labels except one more. I want to automatically add an extra node to the trained network and continue training on this new dataset. How can I do this?
class FeedForewardNN(nn.Module):
    def __init__(self, input_size, h1_size = 264, h2_size = 128, num_services=8):
        super().__init__()
        self.input_size = input_size
        self.lin1 = nn.Linear(input_size, h1_size)
        self.lin2 = nn.Linear(h1_size, h2_size)
        self.lin3 = nn.Linear(h2_size, num_services)
        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        x = self.lin1(x)
        x = self.relu(x)
        x = self.lin2(x)
        x = self.relu(x)
        x = self.lin3(x)
        x = self.sigmoid(x)
        return x

This is the architecture of the feedforward Neural Network.
Then I first train on the data set with only 7 classes.
#Create NN
input_size = len(x_columns)
net1 = FeedForewardNN(input_size, num_services=7)
alpha= 0.001

#Define optimizer
optimizer = optim.Adam(net.parameters(), lr=alpha)
criterion = nn.BCELoss()
running_loss = 0

#Training Loop
loss_list = []
auc_list = []

for i in range(len(train_data_x)):
    optimizer.zero_grad()

    outputs = net1(train_data_x[i])
    loss = criterion(outputs, train_data_y[i])
    loss.backward()
    optimizer.step()

However then, I want to add one additional output node, define the new weights but maintain the old trained weights, and train on this new data set.

Output:
I suggest to replace layer with new one, having desired shape, and than partially assign its parameter values with old ones as follows:
def increaseClassifier( m: torch.nn.Linear ):
    w = m.weight
    b = m.bias
    old_shape = m.weight.shape

    m2 = nn.Linear( old_shape[1], old_shape[0] +1 )
    m2.weight = nn.parameter.Parameter( torch.cat( (m.weight, m2.weight[0:1]) ) )
    m2.bias = nn.parameter.Parameter( torch.cat( (m.bias, m2.bias[0:1]) ) )
    return m2

class FeedForewardNN(nn.Module):
    ...
    def incrHere(self):
        self.lin3 = increaseClassifier( self.lin3 )

UPD:

Can you explain, how these additional weights that come with this new output node are initialized?

The initial weights for new channel come from new layer creation, layer constructor make new parameters with some random initialization, then we are replace part of it with trained weight, and remained part is ready for new training.
m2.weight = nn.parameter.Parameter( torch.cat( (m.weight, m2.weight[0:1]) ) )


Instruction: What's the correct way to implement convolutional blocks of specified depth?

Input:
I'm trying to implement bayesian optimization with Convolutional Neural Netowrk in PyTorch, specifically, I'm trying to translate network structure from Matlab BayesOptExperiment to PyTorch. I want my network to have the following structure:
Input data -> convblock -> maxpool -> convblock -> maxpool -> convblock -> avgpool -> flatten -> linear -> softmax
where convblock consists of:
[conv2Dlayer -> batch normalization layer -> ReLU],
repeated a few times. Current version works as expected only when section_depth = 1 achieving accuracy of around 65-70%, although if I raise the depth of convblock accuracy plummets to around 10%. I'm definitely missing something, but I'm not sure what is that. The structure of my network:
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

#...

class Net(nn.Module):
    def __init__(self, section_depth):
        super().__init__()
        #! define network architecture
        self.section_depth = section_depth
        self.num_filters = round(16/np.sqrt(self.section_depth))
        self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2)
        self.avgpool = nn.AvgPool2d(kernel_size=8)
        self.block1 = nn.ModuleList()
        self.block2 = nn.ModuleList()
        self.block3 = nn.ModuleList()
        self.batchnorm1 = nn.BatchNorm2d(self.num_filters)
        self.batchnorm2 = nn.BatchNorm2d(2*self.num_filters)
        self.batchnorm3 = nn.BatchNorm2d(4*self.num_filters)
        for i in range(self.section_depth):
            channels1 = 3 if i==0 else self.num_filters
            channels2 = self.num_filters if i == 0 else 2*self.num_filters
            channels3 = 2*self.num_filters if i == 0 else 4*self.num_filters
            self.block1.append(nn.Conv2d(in_channels=channels1, out_channels=self.num_filters, kernel_size=3, padding='same'))
            self.block2.append(nn.Conv2d(in_channels=channels2, out_channels=2*self.num_filters, kernel_size=3, padding='same'))
            self.block3.append(nn.Conv2d(in_channels=channels3, out_channels=4*self.num_filters, kernel_size=3, padding='same'))
        self.fc1 = nn.Linear(4*self.num_filters, 10)  # ? number of outputs
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):
        for i in self.block1:
            x = F.relu(self.batchnorm1(i(x)))
        x = self.maxpool(x)
        for i in self.block2:
            x = F.relu(self.batchnorm2(i(x)))
        x = self.maxpool(x)
        for i in self.block3:
            x = F.relu(self.batchnorm3(i(x)))
        x = self.avgpool(x)
        x = torch.flatten(x, 1) # flatten all dimensions except batch
        x = self.fc1(x)
        x = self.softmax(x)
        return x

Any help would be appreciated.

Output: Ok, I figured it out. Turns out that batch normalization layer has learnable parameters, so instead of using the same layer for the whole convblock I had to make a separate batchnorm layer for every convolutional layer.


Instruction: RuntimeError: The size of tensor a (10) must match the size of tensor b (3) at non-singleton dimension 0

Input:
I am using this intersection over union code to determine IOU from my predictions and targets:
def intersection_over_union(boxes_preds, boxes_labels):
    """
    Calculates intersection over union
    Parameters:
        boxes_preds (tensor): Predictions of Bounding Boxes (BATCH_SIZE, 4)
        boxes_labels (tensor): Correct labels of Bounding Boxes (BATCH_SIZE, 4)
        box_format (str): midpoint/corners, if boxes (x,y,w,h) or (x1,y1,x2,y2)
    Returns:
        tensor: Intersection over union for all examples
    """

    box1_x1 = boxes_preds[..., 0:1]
    box1_y1 = boxes_preds[..., 1:2]
    box1_x2 = boxes_preds[..., 2:3]
    box1_y2 = boxes_preds[..., 3:4]  # (N, 1)
    box2_x1 = boxes_labels[..., 0:1]
    box2_y1 = boxes_labels[..., 1:2]
    box2_x2 = boxes_labels[..., 2:3]
    box2_y2 = boxes_labels[..., 3:4]

    x1 = torch.max(box1_x1, box2_x1)
    y1 = torch.max(box1_y1, box2_y1)
    x2 = torch.min(box1_x2, box2_x2)
    y2 = torch.min(box1_y2, box2_y2)

    # .clamp(0) is for the case when they do not intersect
    intersection = (x2 - x1).clamp(0) * (y2 - y1).clamp(0)

    box1_area = abs((box1_x2 - box1_x1) * (box1_y2 - box1_y1))
    box2_area = abs((box2_x2 - box2_x1) * (box2_y2 - box2_y1))

    return intersection / (box1_area + box2_area - intersection + 1e-6)

My inputs looks like this:
My target bounding boxes look like this:
print(targets[0]['boxes'])
tensor([[217., 481., 249., 511.],
        [435., 191., 467., 223.],
        [471.,  86., 503., 118.]])

And my prediction bounding boxes look like this:
predictions['boxes']
tensor([[ 29.7859, 354.9666,  63.0900, 387.6363],
        [469.1072,  85.6840, 503.1974, 119.7137],
        [ 89.3957, 314.1584, 123.9789, 347.1621],
        [432.2971, 188.4454, 468.4712, 227.3808],
        [214.5407, 482.0136, 248.7030, 512.0000],
        [329.1979, 340.8802, 366.3720, 375.8683],
        [298.5089,  99.0098, 334.4280, 129.4205],
        [  0.0000, 347.7724,  17.3409, 384.5709],
        [485.4312, 181.3882, 512.0000, 213.2009],
        [144.5959, 356.5197, 183.4489, 387.4958]])

However, when I apply the IOU function:
iou = intersection_over_union(predictions['boxes'], targets[0]['boxes'])

I get this error:
RuntimeError: The size of tensor a (10) must match the size of tensor b (3) at non-singleton dimension 0

I'm not sure how I can fix the function as I'm guessing this means I have more predictions than targets...

Output:
I've instead opted for torchvisions IOU example:
iou = torchvision.ops.box_iou(predictions['boxes'], targets[0]['boxes'])

There are no errors here.
I alternatively found this implementation (from https://github.com/amdegroot/ssd.pytorch/blob/master/layers/box_utils.py#L48):

def intersect(box_a, box_b):
    """ We resize both tensors to [A,B,2] without new malloc:
    [A,2] -> [A,1,2] -> [A,B,2]
    [B,2] -> [1,B,2] -> [A,B,2]
    Then we compute the area of intersect between box_a and box_b.
    Args:
      box_a: (tensor) bounding boxes, Shape: [A,4].
      box_b: (tensor) bounding boxes, Shape: [B,4].
    Return:
      (tensor) intersection area, Shape: [A,B].
    """
    A = box_a.size(0)
    B = box_b.size(0)
    max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2),
                       box_b[:, 2:].unsqueeze(0).expand(A, B, 2))
    min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2),
                       box_b[:, :2].unsqueeze(0).expand(A, B, 2))
    inter = torch.clamp((max_xy - min_xy), min=0)
    return inter[:, :, 0] * inter[:, :, 1]

def jaccard(box_a, box_b):
    """Compute the jaccard overlap of two sets of boxes.  The jaccard overlap
    is simply the intersection over union of two boxes.  Here we operate on
    ground truth boxes and default boxes.
    E.g.:
        A ∩ B / A ∪ B = A ∩ B / (area(A) + area(B) - A ∩ B)
    Args:
        box_a: (tensor) Ground truth bounding boxes, Shape: [num_objects,4]
        box_b: (tensor) Prior boxes from priorbox layers, Shape: [num_priors,4]
    Return:
        jaccard overlap: (tensor) Shape: [box_a.size(0), box_b.size(0)]
    """
    inter = intersect(box_a, box_b)
    area_a = ((box_a[:, 2]-box_a[:, 0]) *
              (box_a[:, 3]-box_a[:, 1])).unsqueeze(1).expand_as(inter)  # [A,B]
    area_b = ((box_b[:, 2]-box_b[:, 0]) *
              (box_b[:, 3]-box_b[:, 1])).unsqueeze(0).expand_as(inter)  # [A,B]
    union = area_a + area_b - inter
    return inter / union  # [A,B]

Where applying jaccard produces the same output as torchvisions own functions:
iou = jaccard(predictions['boxes'], targets[0]['boxes'])

An example tensor when printing IOU:
tensor([[0.0000, 0.0000],
        [0.0000, 0.0000],
        [0.0000, 0.0000],
        [0.0000, 0.0000],
        [0.0000, 0.9322],
        [0.8021, 0.0000],
        [0.0000, 0.0000],
        [0.0000, 0.0000]])


Instruction: is there any way to include a counter(a variable that count something) in a loss function in pytorch?

Input:
These are some lines from my loss function. output is the output of a multiclass classification network.
bin_count=torch.bincount(torch.where(output>.1)[0], minlength=output.shape[0])

dr_output = (bin_count == 1) & (torch.argmax(output, dim=1)==labels)

I want dr_output.sum() to be part of my loss function. But there are many limitations in my implementation. Some functions are non-differentiable in pytorch, and also dr_output may be zero which is also not allowed if I only use dr_output as my loss. Can anyone please suggest to me a way around these problems?

Output:
If I got it correctly:
bin_count=torch.bincount(torch.where(output>.1)[0], minlength=output.shape[0])

computes how many elements are greater than .1, for each row.
Instead:
dr_output = (bin_count == 1) & (torch.argmax(output, dim=1)==labels)

is true if there is only an element greater than .1 in the corresponding row, and the prediction is correct.
dr_output.sum() then counts how many rows verify this condition, so minimizing the loss may enforce incorrect predictions or distributions with more values greater than .1.
Given these considerations, you can approximate your loss with the following:
import torch.nn.functional as F

# x are the inputs, y the labels

mask = x > 0.1
p = F.softmax(x, dim=1)
out = p * (mask.sum(dim=1, keepdim=True) == 1)

loss = out[torch.arange(x.shape[0]), y].sum()

You can devise similar variants that are more fit for your problem.


Instruction: Are there benefits to having Actor and Critic use significantly different models?

Input:
In Actor-Critic methods the Actor and Critic are assigned two complimentary, but different goals. I'm trying to understand whether the differences between these goals (updating a policy and updating a value function) are large enough to warrant different models for the Actor and Critic, or if they are of similar enough complexity that the same model should be reused for simplicity. I realize that this could be very situational, but not in what way. For example, does the balance shift as the model complexity grows?
Please let me know if there are any rules of thumb for this, or if you know of a specific publication that addresses the issue.

Output:
The empirical results suggest the exact opposite - that it is important to have the same network doing both (up to some final layer/head). The main reason for this is that learning value network (critis) provides signal for shaping represntation of the policy (actor) that otherwise would be nearly impossible to get.
In fact if you think about these, these are extremely similar goals, since for optimal deterministic policy
pi(s) = arg max_a Q(s, a) = arg max_a V(T(s, a))

where T is the transition dynamics.


Instruction: How to load data from multiply datasets in pytorch

Input:
I have two datasets of images - indoors and outdoors, they don't have the same number of examples.
Each dataset has images that contain a certain number of classes (minimum 1 maximum 4), these classes can appear in both datasets, and each class has 4 categories - red, blue, green, white.
Example:
Indoor - cats, dogs, horses
Outdoor - dogs, humans
I am trying to train a model, where I tell it, "here is an image that contains a cat, tell me it's color" regardless of where it was taken (Indoors, outdoors, In a car, on the moon)
To do that,
I need to present my model examples so that every batch has only one category (cat, dog, horse or human), but I want to sample from all datasets (two in this case) that contains these objects and mix them. How can I do this?
It has to take into account that the number of examples in each dataset is different, and that some categories appear in one dataset where others can appear in more than one.
and each batch must contain only one category.
I would appreciate any help, I have been trying to solve this for a few days now.

Output:
Assuming the question is:

Combine 2+ data sets with potentially overlapping categories of objects (distinguishable by label)
Each object has 4 "subcategories" for each color (distinguishable by label)
Each batch should only contain a single object category

The first step will be to ensure consistency of the object labels from both data sets, if not already consistent. For example, if the dog class is label 0 in the first data set but label 2 in the second data set, then we need to make sure the two dog categories are correctly merged. We can do this "translation" with a simple data set wrapper:
class TranslatedDataset(Dataset):
  """
  Args:
    dataset: The original dataset.
    translate_label: A lambda (function) that maps the original
      dataset label to the label it should have in the combined data set
  """
  def __init__(self, dataset, translate_label):
    super().__init__()
    self._dataset = dataset
    self._translate_label = translate_label

  def __len__(self):
    return len(self._dataset)

  def __getitem__(self, idx):
    inputs, target = self._dataset[idx]
    return inputs, self._translate_label(target)

The next step is combining the translated data sets together, which can be done easily with a ConcatDataset:
first_original_dataset = ...
second_original_dataset = ...

first_translated = TranslateDataset(
  first_original_dataset,
  lambda y: 0 if y is 2 else 2 if y is 0 else y, # or similar
)
second_translated = TranslateDataset(
  second_original_dataset,
  lambda y: y, # or similar
)

combined = ConcatDataset([first_translated, second_translated])

Finally, we need to restrict batch sampling to the same class, which is possible with a custom Sampler when creating the data loader.
class SingleClassSampler(torch.utils.data.Sampler):
  def __init__(self, dataset, batch_size):
    super().__init__()
    # We need to create sequential groups
    # with batch_size elements from the same class
    indices_for_target = {} # dict to store a list of indices for each target

    for i, (_, target) in enumerate(dataset):
      # converting to string since Tensors hash by reference, not value
      str_targ = str(target)
      if str_targ not in indices_for_target:
        indices_for_target[str_targ] = []
      indices_for_target[str_targ] += [i]

    # make sure we have a whole number of batches for each class
    trimmed = {
      k: v[:-(len(v) % batch_size)]
      for k, v in indices_for_target.items()
    }

    # concatenate the lists of indices for each class
    self._indices = sum(list(trimmed.values()))

  def __len__(self):
    return len(self._indices)

  def __iter__(self):
    yield from self._indices

Then to use the sampler:
loader = DataLoader(
  combined,
  sampler=SingleClassSampler(combined, 64),
  batch_size=64,
  shuffle=True
)

I haven't run this code, so it might not be exactly right, but hopefully it will put you on the right track.

torch.utils.data Docs


Instruction: RuntimeError: mat1 and mat2 shapes cannot be multiplied (25x340 and 360x1)

Input:
I get this error message and I'm not sure why. My input is (batch, 1, 312) from tabular data and this CNN is constructed for a regression prediction. I worked out the shapes for each step with the formula (input + 2*padding - filter size)/stride + 1 as in the comment below. The problem appears to occur at  x = self.fc(x) and I can't figure out why. Your help is greatly appreciated. Thank you.
class CNNWeather(nn.Module):
    # input (batch, 1, 312)
    def __init__(self):
        super(CNNWeather, self).__init__()
        self.conv1 = nn.Conv1d(in_channels=1, out_channels=8, kernel_size=9, stride=1, padding='valid')         # (312+2*0-9)/1 + 1 = 304
        self.pool1 = nn.AvgPool1d(kernel_size=2, stride=2)                                                      # 304/2 = 302
        self.conv2 = nn.Conv1d(in_channels=8, out_channels=12, kernel_size=3, stride=1, padding='valid')        # (302-3)/1+1 = 300
        self.pool2 = nn.AvgPool1d(kernel_size=2, stride=2)                                                      # 300/2 = 150
        self.conv3 = nn.Conv1d(in_channels=12, out_channels=16, kernel_size=3, stride=1, padding='valid')       # (150-3)/1+1 = 76
        self.pool3 = nn.AvgPool1d(kernel_size=2, stride=2)                                                      # 76/2 = 38
        self.conv4 = nn.Conv1d(in_channels=16, out_channels=20, kernel_size=3, stride=1, padding='valid')       # (38-3)/1+1 = 36
        self.pool4 = nn.AvgPool1d(kernel_size=2, stride=2)                                                      # 36/2 = 18  (batch, 20, 18)
        self.fc = nn.Linear(in_features=20*18, out_features=1)

    def forward(self, x):
        x = self.pool1(F.relu(self.conv1(x)))
        x = self.pool2(F.relu(self.conv2(x)))
        x = self.pool3(F.relu(self.conv3(x)))
        x = self.pool4(F.relu(self.conv4(x)))
        print(x.size())
        x = x.view(x.size(0), -1)               # flatten   (batch, 20*18)
        x = self.fc(x)
        return x

Output:
The problem seems to be related to the input size of your FC layer:
self.fc = nn.Linear(in_features=20*18, out_features=1)

The output of the previous layer is 340, so you must use in_features=340.
These are the shapes of the output for the third and fourth layers.
torch.Size([5, 16, 73]) conv3 out
torch.Size([5, 16, 36]) pool3 out
torch.Size([5, 20, 34]) conv4 out
torch.Size([5, 20, 17]) pool4 out

Notice that out of the "pool4" layer come 20x17, meaning 340 elements.


Instruction: Pytorch: layer not transferred on GPU with to() function

Input:
In the following code, I expected tensor x and layer l both on GPU, instead only the the tensor x results to be on the GPU, and not the layer l. In fact, using this approach results in RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! during a learning stage.
import torch
x = torch.zeros(1)
x = x.to('cuda')
try:
    x.get_device()
    print('x: gpu')
except:
    print('x:','cpu')

l = torch.nn.Linear(1,1)
l = l.to('cuda')
try:
    l.get_device()
    print('l: gpu')
except:
    print('l:','cpu')

the output is:
x: gpu
l: cpu

instead of both gpu.
why this?
Torch version: 1.10.2+cu113

Output:
you can't call .get_device() on a nn.Linear object, therefore your second try block fails and it prints the code in the exception part. In order to check what device your module is on you can do the following:
print(next(l.parameters()).device)

output:
>> device(type='cuda', index=0)


Instruction: Result type cast error when doing calculations with Pytorch model parameters

Input:
When I ran the code below:
import torchvision

model = torchvision.models.densenet201(num_classes=10)
params = model.state_dict()
for var in params:
    params[var] *= 0.1

a RuntimeError was reported:
RuntimeError: result type Float can't be cast to the desired output type Long

But when I changed params[var] *= 0.1 to params[var] = params[var] * 0.1, the error disappears.
Why would this happen?
I thought params[var] *= 0.1 had the same effect as params[var] = params[var] * 0.1.

Output:
First, let us know the first long-type parameter in densenet201, you will find the features.norm0.num_batches_tracked which indicates the number of mini-batches during training used to calculate the mean and variance if there is BatchNormalization layer in the model. This parameter is a long-type number and cannot be float type because it behaves like a counter.
Second, in PyTorch, there are two types of operations:

Non-Inplace operations: you assign the new output after calculation to a new copy from the variable, e.g. x = x + 1 or x = x / 2. The memory location of x before assignment not equal to the memory location after assignment because you have a copy from the original variable.
Inplace operations: when the calculations directly applied to the original copy of the variable without making any copy here e.g. x += 1 or x /= 2.

Let's move to your example to understand what happened:

Non-Inplcae operation:
model = torchvision.models.densenet201(num_classes=10)
params = model.state_dict()
name = 'features.norm0.num_batches_tracked'

print(id(params[name]))  # 140247785908560
params[name] = params[name] + 0.1
print(id(params[name]))  # 140247785908368
print(params[name].type()) # changed to torch.FloatTensor

Inplace operation:
print(id(params[name]))  # 140247785908560
params[name] += 1
print(id(params[name]))  # 140247785908560
print(params[name].type()) # still torch.LongTensor

params[name] += 0.1     # you want to change the original copy type to float ,you got an error

Finally, some remarks:

In-place operations save some memory, but can be problematic when computing derivatives because of an immediate loss of history. Hence, their use is discouraged. Source
You should be cautious when you decide to use in-place operations since they overwrite the original content.
If you use pandas, this is a bit similar to the inplace=True in pandas :).

This is a good resource to read more about in-place operation source and read also this discussion source.


Instruction: How can I save the path of an image (string) from my dataloader (PyTorch)?

Input:
I've created a dataloader for my object detection task.
However, I cannot place the image/path name to a tensor. Instead I have it indexed, where in the last portion of the dataloader class, I have this:
target = {}
        target['boxes'] = boxes
        target['labels'] = labels
        target['image_id'] = torch.tensor([index])
        target['area'] = area
        target['iscrowd'] = iscrowd
        target['image_name'] = torch.tensor(index)

        return image, target

where atm image_id and image_name are the same thing.
When I print out the image_name from the dataloader, I of course get this:
for image, target in valid_data_loader:
  print(target[0]['image_name'])

Output:

tensor(0)
tensor(1)
tensor(2)
tensor(3)
tensor(4)
tensor(5)
tensor(6)
tensor(7)

I'm aware that strings can't be saved into torch tensors, so is there any way I can refer back to the original image name rather than the index of the tensor? Or would I just have to use the number that comes out and refer back to the dataset class (not dataloader)?
I ultimately want to save the image name, and attributes such as bounding box info to a separate numpy dataframe.

Output:
Ok, so this is a bit ad-hoc and not exactly what I was thinking but here is one method I have used to retrieve the paths/image names. I basically find the id from the dataloader by removing it from the tensor. I then use the tensor_id to find the corresponding id in the original dataframe:
for image, target in valid_data_loader:
  tensor_id = target[0]['image_name'].item()
  print(valid_df.iloc[tensor_id]['image_id'])

I don't know if this is efficient though but it got what I wanted...


Instruction: Is there a function to divide a n dimensional tensor with an n-1 dimensional tensor

Input:
I am not sure how to word this question properly so I will show some examples to describe the desired behaviour.
I'm looking to divide a tensor in this specific way.

Divide a vector by 1 scalar, for example [1, 2, 3, 4, 5] divided by 2 = [0.5, 1, 1.5, 2, 2.5]

Divide a matrix by 2 scalars, for example [[1, 2, 3], [2, 3, 4]] by [2, 4] = [[0.5, 1, 1.5], [0.5, 0.75, 1]]

Divide a 3 dimensional tensor by a 2 dimensional tensor, for example [[[1, 2, 3], [2, 3, 4]], [[4,5,6], [7,8,9]], [[9,8,6], [1,2,3]]] divided by [[1, 2], [3, 4], [5, 6]] = [[[1, 2, 3], [1, 1.5, 2]], [[4/3,5/3,2], [7/4,2,9/4]], [[9/5,8/5,6/5], [1/6,2/6,3/6]]]

Divide a N dimensional tensor by a N-1 dimensional tensor...

I'm looking for a pytorch way to do this.

Output:
You can expand the dimensions of the N-1 dimensional tensor to make it broadcastable with the N-dimensional tensor.
tensor_a / tensor_b.unsqueeze(-1)

This generalizes, even when the denominator is a scalar. The -1 dimension means the last dimension. This follows from python indexing rules, in which sequence[-1] gives you the last element of the sequence.
a = torch.as_tensor([1, 2, 3, 4, 5])
b = torch.as_tensor(2)
a / b.unsqueeze(-1)
# tensor([0.5000, 1.0000, 1.5000, 2.0000, 2.5000])

a = torch.as_tensor([[1, 2, 3], [2, 3, 4]])
b = torch.as_tensor([2, 4])
a / b.unsqueeze(-1)
# tensor([[0.5000, 1.0000, 1.5000],
#         [0.5000, 0.7500, 1.0000]])

a = torch.as_tensor([[[1, 2, 3], [2, 3, 4]], [[4,5,6], [7,8,9]], [[9,8,6], [1,2,3]]])
b = torch.as_tensor([[1, 2], [3, 4], [5, 6]])
a / b.unsqueeze(-1)
# tensor([[[1.0000, 2.0000, 3.0000],
#          [1.0000, 1.5000, 2.0000]],
#
#         [[1.3333, 1.6667, 2.0000],
#          [1.7500, 2.0000, 2.2500]],
#
#         [[1.8000, 1.6000, 1.2000],
#         [0.1667, 0.3333, 0.5000]]])