Instruction: DCGAN debugging. Getting just garbage

Input:
Introduction:
I am trying to get a CDCGAN (Conditional Deep Convolutional Generative Adversarial Network) to work on the MNIST dataset which should be fairly easy considering that the library (PyTorch) I am using has a tutorial on its website.
But I can't seem to get It working it just produces garbage or the model collapses or both.
What I tried:

making the model Conditional semi-supervised learning
using batch norm
using dropout on each layer besides the input/output layer on the generator and discriminator
label smoothing to combat overconfidence
adding noise to the images (I guess you call this instance noise) to get a better data distribution
use leaky relu to avoid vanishing gradients
using a replay buffer to combat forgetting of learned stuff and overfitting
playing with hyperparameters
comparing it to the model from PyTorch tutorial
basically what I did besides some things like Embedding layer ect.

Images my Model generated:
Hyperparameters:
batch_size=50, learning_rate_discrimiantor=0.0001, learning_rate_generator=0.0003, shuffle=True, ndf=64, ngf=64, droupout=0.5

batch_size=50, learning_rate_discriminator=0.0003, learning_rate_generator=0.0003, shuffle=True, ndf=64, ngf=64, dropout=0

Images Pytorch tutorial Model generated:
Code for the pytorch tutorial dcgan model
As comparison here are the images from the DCGAN from the pytorch turoial:

My Code:
import torch
import torch.nn as nn
import torchvision
from torchvision import transforms, datasets
import torch.nn.functional as F
from torch import optim as optim
from torch.utils.tensorboard import SummaryWriter

import numpy as np

import os
import time

class Discriminator(torch.nn.Module):
    def __init__(self, ndf=16, dropout_value=0.5):  # ndf feature map discriminator
        super().__init__()
        self.ndf = ndf
        self.droupout_value = dropout_value

        self.condi = nn.Sequential(
            nn.Linear(in_features=10, out_features=64 * 64)
        )

        self.hidden0 = nn.Sequential(
            nn.Conv2d(in_channels=2, out_channels=self.ndf, kernel_size=4, stride=2, padding=1, bias=False),
            nn.LeakyReLU(0.2),
        )
        self.hidden1 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf, out_channels=self.ndf * 2, kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ndf * 2),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.hidden2 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 2, out_channels=self.ndf * 4, kernel_size=4, stride=2, padding=1, bias=False),
            #nn.BatchNorm2d(self.ndf * 4),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.hidden3 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 4, out_channels=self.ndf * 8, kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ndf * 8),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.out = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 8, out_channels=1, kernel_size=4, stride=1, padding=0, bias=False),
            torch.nn.Sigmoid()
        )

    def forward(self, x, y):
        y = self.condi(y.view(-1, 10))
        y = y.view(-1, 1, 64, 64)

        x = torch.cat((x, y), dim=1)

        x = self.hidden0(x)
        x = self.hidden1(x)
        x = self.hidden2(x)
        x = self.hidden3(x)
        x = self.out(x)

        return x

class Generator(torch.nn.Module):
    def __init__(self, n_features=100, ngf=16, c_channels=1, dropout_value=0.5):  # ngf feature map of generator
        super().__init__()
        self.ngf = ngf
        self.n_features = n_features
        self.c_channels = c_channels
        self.droupout_value = dropout_value

        self.hidden0 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.n_features + 10, out_channels=self.ngf * 8,
                               kernel_size=4, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(self.ngf * 8),
            nn.LeakyReLU(0.2)
        )

        self.hidden1 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 8, out_channels=self.ngf * 4,
                               kernel_size=4, stride=2, padding=1, bias=False),
            #nn.BatchNorm2d(self.ngf * 4),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.hidden2 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 4, out_channels=self.ngf * 2,
                               kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ngf * 2),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.hidden3 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 2, out_channels=self.ngf,
                               kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ngf),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.out = nn.Sequential(
            # "out_channels=1" because gray scale
            nn.ConvTranspose2d(in_channels=self.ngf, out_channels=1, kernel_size=4,
                               stride=2, padding=1, bias=False),
            nn.Tanh()
        )

    def forward(self, x, y):
        x_cond = torch.cat((x, y), dim=1)  # Combine flatten image with conditional input (class labels)

        x = self.hidden0(x_cond)           # Image goes into a "ConvTranspose2d" layer
        x = self.hidden1(x)
        x = self.hidden2(x)
        x = self.hidden3(x)
        x = self.out(x)

        return x

class Logger:
    def __init__(self, model_name, model1, model2, m1_optimizer, m2_optimizer, model_parameter, train_loader):
        self.out_dir = "data"
        self.model_name = model_name
        self.train_loader = train_loader
        self.model1 = model1
        self.model2 = model2
        self.model_parameter = model_parameter
        self.m1_optimizer = m1_optimizer
        self.m2_optimizer = m2_optimizer

        # Exclude Epochs of the model name. This make sense e.g. when we stop a training progress and continue later on.
        self.experiment_name = '_'.join("{!s}={!r}".format(k, v) for (k, v) in model_parameter.items())\
            .replace("Epochs" + "=" + str(model_parameter["Epochs"]), "")

        self.d_error = 0
        self.g_error = 0

        self.tb = SummaryWriter(log_dir=str(self.out_dir + "/log/" + self.model_name + "/runs/" + self.experiment_name))

        self.path_image = os.path.join(os.getcwd(), f'{self.out_dir}/log/{self.model_name}/images/{self.experiment_name}')
        self.path_model = os.path.join(os.getcwd(), f'{self.out_dir}/log/{self.model_name}/model/{self.experiment_name}')

        try:
            os.makedirs(self.path_image)
        except Exception as e:
            print("WARNING: ", str(e))

        try:
            os.makedirs(self.path_model)
        except Exception as e:
            print("WARNING: ", str(e))

    def log_graph(self, model1_input, model2_input, model1_label, model2_label):
        self.tb.add_graph(self.model1, input_to_model=(model1_input, model1_label))
        self.tb.add_graph(self.model2, input_to_model=(model2_input, model2_label))

    def log(self, num_epoch, d_error, g_error):
        self.d_error = d_error
        self.g_error = g_error

        self.tb.add_scalar("Discriminator Train Error", self.d_error, num_epoch)
        self.tb.add_scalar("Generator Train Error", self.g_error, num_epoch)

    def log_image(self, images, epoch, batch_num):
        grid = torchvision.utils.make_grid(images)
        torchvision.utils.save_image(grid, f'{self.path_image}\\Epoch_{epoch}_batch_{batch_num}.png')

        self.tb.add_image("Generator Image", grid)

    def log_histogramm(self):
        for name, param in self.model2.named_parameters():
            self.tb.add_histogram(name, param, self.model_parameter["Epochs"])
            self.tb.add_histogram(f'gen_{name}.grad', param.grad, self.model_parameter["Epochs"])

        for name, param in self.model1.named_parameters():
            self.tb.add_histogram(name, param, self.model_parameter["Epochs"])
            self.tb.add_histogram(f'dis_{name}.grad', param.grad, self.model_parameter["Epochs"])

    def log_model(self, num_epoch):
        torch.save({
            "epoch": num_epoch,
            "model_generator_state_dict": self.model1.state_dict(),
            "model_discriminator_state_dict": self.model2.state_dict(),
            "optimizer_generator_state_dict":  self.m1_optimizer.state_dict(),
            "optimizer_discriminator_state_dict":  self.m2_optimizer.state_dict(),
        }, str(self.path_model + f'\\{time.time()}_epoch{num_epoch}.pth'))

    def close(self, logger, images, num_epoch,  d_error, g_error):
        logger.log_model(num_epoch)
        logger.log_histogramm()
        logger.log(num_epoch, d_error, g_error)
        self.tb.close()

    def display_stats(self, epoch, batch_num, dis_error, gen_error):
        print(f'Epoch: [{epoch}/{self.model_parameter["Epochs"]}] '
              f'Batch: [{batch_num}/{len(self.train_loader)}] '
              f'Loss_D: {dis_error.data.cpu()}, '
              f'Loss_G: {gen_error.data.cpu()}')

def get_MNIST_dataset(num_workers_loader, model_parameter, out_dir="data"):
    compose = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.CenterCrop((64, 64)),
        transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.5], std=[0.5])
    ])

    dataset = datasets.MNIST(
        root=out_dir,
        train=True,
        download=True,
        transform=compose
    )

    train_loader = torch.utils.data.DataLoader(dataset,
                                               batch_size=model_parameter["batch_size"],
                                               num_workers=num_workers_loader,
                                               shuffle=model_parameter["shuffle"])

    return dataset, train_loader

def train_discriminator(p_optimizer, p_noise, p_images, p_fake_target, p_real_target, p_images_labels, p_fake_labels, device):
    p_optimizer.zero_grad()

    # 1.1 Train on real data
    pred_dis_real = discriminator(p_images, p_images_labels)
    error_real = loss(pred_dis_real, p_real_target)

    error_real.backward()

    # 1.2 Train on fake data
    fake_data = generator(p_noise, p_fake_labels).detach()
    fake_data = add_noise_to_image(fake_data, device)
    pred_dis_fake = discriminator(fake_data, p_fake_labels)
    error_fake = loss(pred_dis_fake, p_fake_target)

    error_fake.backward()

    p_optimizer.step()

    return error_fake + error_real

def train_generator(p_optimizer, p_noise, p_real_target, p_fake_labels, device):
    p_optimizer.zero_grad()

    fake_images = generator(p_noise, p_fake_labels)
    fake_images = add_noise_to_image(fake_images, device)
    pred_dis_fake = discriminator(fake_images, p_fake_labels)
    error_fake = loss(pred_dis_fake, p_real_target)  # because
    """
    We use "p_real_target" instead of "p_fake_target" because we want to
    maximize that the discriminator is wrong.
    """

    error_fake.backward()

    p_optimizer.step()

    return fake_images, pred_dis_fake, error_fake

# TODO change to a Truncated normal distribution
def get_noise(batch_size, n_features=100):
    return torch.FloatTensor(batch_size, n_features, 1, 1).uniform_(-1, 1)

# We flip label of real and fate data. Better gradient flow I have told
def get_real_data_target(batch_size):
    return torch.FloatTensor(batch_size, 1, 1, 1).uniform_(0.0, 0.2)

def get_fake_data_target(batch_size):
    return torch.FloatTensor(batch_size, 1, 1, 1).uniform_(0.8, 1.1)

def image_to_vector(images):
    return torch.flatten(images, start_dim=1, end_dim=-1)

def vector_to_image(images):
    return images.view(images.size(0), 1, 28, 28)

def get_rand_labels(batch_size):
    return torch.randint(low=0, high=9, size=(batch_size,))

def load_model(model_load_path):
    if model_load_path:
        checkpoint = torch.load(model_load_path)

        discriminator.load_state_dict(checkpoint["model_discriminator_state_dict"])
        generator.load_state_dict(checkpoint["model_generator_state_dict"])

        dis_opti.load_state_dict(checkpoint["optimizer_discriminator_state_dict"])
        gen_opti.load_state_dict(checkpoint["optimizer_generator_state_dict"])

        return checkpoint["epoch"]

    else:
        return 0

def init_model_optimizer(model_parameter, device):
    # Initialize the Models
    discriminator = Discriminator(ndf=model_parameter["ndf"], dropout_value=model_parameter["dropout"]).to(device)
    generator = Generator(ngf=model_parameter["ngf"], dropout_value=model_parameter["dropout"]).to(device)

    # train
    dis_opti = optim.Adam(discriminator.parameters(), lr=model_parameter["learning_rate_dis"], betas=(0.5, 0.999))
    gen_opti = optim.Adam(generator.parameters(), lr=model_parameter["learning_rate_gen"], betas=(0.5, 0.999))

    return discriminator, generator, dis_opti, gen_opti

def get_hot_vector_encode(labels, device):
    return torch.eye(10)[labels].view(-1, 10, 1, 1).to(device)

def add_noise_to_image(images, device, level_of_noise=0.1):
    return images[0].to(device) + (level_of_noise) * torch.randn(images.shape).to(device)

if __name__ == "__main__":
    # Hyperparameter
    model_parameter = {
        "batch_size": 500,
        "learning_rate_dis": 0.0002,
        "learning_rate_gen": 0.0002,
        "shuffle": False,
        "Epochs": 10,
        "ndf": 64,
        "ngf": 64,
        "dropout": 0.5
    }

    # Parameter
    r_frequent = 10        # How many samples we save for replay per batch (batch_size / r_frequent).
    model_name = "CDCGAN"   # The name of you model e.g. "Gan"
    num_workers_loader = 1  # How many workers should load the data
    sample_save_size = 16   # How many numbers your saved imaged should show
    device = "cuda"         # Which device should be used to train the neural network
    model_load_path = ""    # If set load model instead of training from new
    num_epoch_log = 1       # How frequent you want to log/
    torch.manual_seed(43)   # Sets a seed for torch for reproducibility

    dataset_train, train_loader = get_MNIST_dataset(num_workers_loader, model_parameter)  # Get dataset

    # Initialize the Models and optimizer
    discriminator, generator, dis_opti, gen_opti = init_model_optimizer(model_parameter, device)  # Init model/Optimizer

    start_epoch = load_model(model_load_path)  # when we want to load a model

    # Init Logger
    logger = Logger(model_name, generator, discriminator, gen_opti, dis_opti, model_parameter, train_loader)

    loss = nn.BCELoss()

    images, labels = next(iter(train_loader))  # For logging

    # For testing
    # pred = generator(get_noise(model_parameter["batch_size"]).to(device), get_hot_vector_encode(get_rand_labels(model_parameter["batch_size"]), device))
    # dis = discriminator(images.to(device), get_hot_vector_encode(labels, device))

    logger.log_graph(get_noise(model_parameter["batch_size"]).to(device), images.to(device),
                     get_hot_vector_encode(get_rand_labels(model_parameter["batch_size"]), device),
                     get_hot_vector_encode(labels, device))

    # Array to store
    exp_replay = torch.tensor([]).to(device)

    for num_epoch in range(start_epoch, model_parameter["Epochs"]):
        for batch_num, data_loader in enumerate(train_loader):
            images, labels = data_loader
            images = add_noise_to_image(images, device)  # Add noise to the images

            # 1. Train Discriminator
            dis_error = train_discriminator(
                                            dis_opti,
                                            get_noise(model_parameter["batch_size"]).to(device),
                                            images.to(device),
                                            get_fake_data_target(model_parameter["batch_size"]).to(device),
                                            get_real_data_target(model_parameter["batch_size"]).to(device),
                                            get_hot_vector_encode(labels, device),
                                            get_hot_vector_encode(
                                                get_rand_labels(model_parameter["batch_size"]), device),
                                            device
                                            )

            # 2. Train Generator
            fake_image, pred_dis_fake, gen_error = train_generator(
                                                                  gen_opti,
                                                                  get_noise(model_parameter["batch_size"]).to(device),
                                                                  get_real_data_target(model_parameter["batch_size"]).to(device),
                                                                  get_hot_vector_encode(
                                                                      get_rand_labels(model_parameter["batch_size"]),
                                                                      device),
                                                                  device
                                                                  )

            # Store a random point for experience replay
            perm = torch.randperm(fake_image.size(0))
            r_idx = perm[:max(1, int(model_parameter["batch_size"] / r_frequent))]
            r_samples = add_noise_to_image(fake_image[r_idx], device)
            exp_replay = torch.cat((exp_replay, r_samples), 0).detach()

            if exp_replay.size(0) >= model_parameter["batch_size"]:
                # Train on experienced data
                dis_opti.zero_grad()

                r_label = get_hot_vector_encode(torch.zeros(exp_replay.size(0)).numpy(), device)
                pred_dis_real = discriminator(exp_replay, r_label)
                error_real = loss(pred_dis_real,  get_fake_data_target(exp_replay.size(0)).to(device))

                error_real.backward()

                dis_opti.step()

                print(f'Epoch: [{num_epoch}/{model_parameter["Epochs"]}] '
                      f'Batch: Replay/Experience batch '
                      f'Loss_D: {error_real.data.cpu()}, '
                      )

                exp_replay = torch.tensor([]).to(device)

            logger.display_stats(epoch=num_epoch, batch_num=batch_num, dis_error=dis_error, gen_error=gen_error)

            if batch_num % 100 == 0:
                logger.log_image(fake_image[:sample_save_size], num_epoch, batch_num)

        logger.log(num_epoch, dis_error, gen_error)
        if num_epoch % num_epoch_log == 0:
            logger.log_model(num_epoch)
            logger.log_histogramm()
    logger.close(logger, fake_image[:sample_save_size], num_epoch, dis_error, gen_error)

First link to my Code (Pastebin)
Second link to my Code (0bin)
Conclusion:
Since I implemented all these things (e.g. label smoothing) which are considered beneficial to a GAN/DCGAN.
And my Model still performs worse than the Tutorial DCGAN from PyTorch I think I might have a bug in my code but I can't seem to find it.
Reproducibility:
You should be able to just copy the code and run it if you have the libraries that I imported installed to look for yourself if you can find anything.
I appreciate any feedback.

Output:
So I solved this issue a while ago, but forgot to post an answer on stack overflow. So I will simply post my code here which should work probably pretty good.
Some disclaimer:

I am not quite sure if it works since I did this a year ago
its for 128x128px Images MNIST
It's not a vanilla GAN I used various optimization techniques
If you want to use it you need to change various details, such as the training dataset

Resources:

Multi-Scale Gradients
Instance Noise
Various tricks I used
More tricks

``
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader

import pytorch_lightning as pl
from pytorch_lightning import loggers

from numpy.random import choice

import os
from pathlib import Path
import shutil

from collections import OrderedDict

# custom weights initialization called on netG and netD
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1.0, 0.02)
        nn.init.constant_(m.bias.data, 0)

# randomly flip some labels
def noisy_labels(y, p_flip=0.05):  # # flip labels with 5% probability
    # determine the number of labels to flip
    n_select = int(p_flip * y.shape[0])
    # choose labels to flip
    flip_ix = choice([i for i in range(y.shape[0])], size=n_select)
    # invert the labels in place
    y[flip_ix] = 1 - y[flip_ix]
    return y

class AddGaussianNoise(object):
    def __init__(self, mean=0.0, std=0.1):
        self.std = std
        self.mean = mean

    def __call__(self, tensor):
        tensor = tensor.cuda()
        return tensor + (torch.randn(tensor.size()) * self.std + self.mean).cuda()

    def __repr__(self):
        return self.__class__.__name__ + '(mean={0}, std={1})'.format(self.mean, self.std)

def resize2d(img, size):
    return (F.adaptive_avg_pool2d(img, size).data).cuda()

def get_valid_labels(img):
    return ((0.8 - 1.1) * torch.rand(img.shape[0], 1, 1, 1) + 1.1).cuda()  # soft labels

def get_unvalid_labels(img):
    return (noisy_labels((0.0 - 0.3) * torch.rand(img.shape[0], 1, 1, 1) + 0.3)).cuda()  # soft labels

class Generator(pl.LightningModule):
    def __init__(self, ngf, nc, latent_dim):
        super(Generator, self).__init__()
        self.ngf = ngf
        self.latent_dim = latent_dim
        self.nc = nc

        self.fc0 = nn.Sequential(
            # input is Z, going into a convolution
            nn.utils.spectral_norm(nn.ConvTranspose2d(latent_dim, ngf * 16, 4, 1, 0, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ngf * 16)
        )

        self.fc1 = nn.Sequential(
            # state size. (ngf*8) x 4 x 4
            nn.utils.spectral_norm(nn.ConvTranspose2d(ngf * 16, ngf * 8, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ngf * 8)
        )

        self.fc2 = nn.Sequential(
            # state size. (ngf*4) x 8 x 8
            nn.utils.spectral_norm(nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ngf * 4)
        )

        self.fc3 = nn.Sequential(
            # state size. (ngf*2) x 16 x 16
            nn.utils.spectral_norm(nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ngf * 2)
        )

        self.fc4 = nn.Sequential(
            # state size. (ngf) x 32 x 32
            nn.utils.spectral_norm(nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ngf)
        )

        self.fc5 = nn.Sequential(
            # state size. (nc) x 64 x 64
            nn.utils.spectral_norm(nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False)),
            nn.Tanh()
        )

        # state size. (nc) x 128 x 128

        # For Multi-Scale Gradient
        # Converting the intermediate layers into images
        self.fc0_r = nn.Conv2d(ngf * 16, self.nc, 1)
        self.fc1_r = nn.Conv2d(ngf * 8, self.nc, 1)
        self.fc2_r = nn.Conv2d(ngf * 4, self.nc, 1)
        self.fc3_r = nn.Conv2d(ngf * 2, self.nc, 1)
        self.fc4_r = nn.Conv2d(ngf, self.nc, 1)

    def forward(self, input):
        x_0 = self.fc0(input)
        x_1 = self.fc1(x_0)
        x_2 = self.fc2(x_1)
        x_3 = self.fc3(x_2)
        x_4 = self.fc4(x_3)
        x_5 = self.fc5(x_4)

        # For Multi-Scale Gradient
        # Converting the intermediate layers into images
        x_0_r = self.fc0_r(x_0)
        x_1_r = self.fc1_r(x_1)
        x_2_r = self.fc2_r(x_2)
        x_3_r = self.fc3_r(x_3)
        x_4_r = self.fc4_r(x_4)

        return x_5, x_0_r, x_1_r, x_2_r, x_3_r, x_4_r

class Discriminator(pl.LightningModule):
    def __init__(self, ndf, nc):
        super(Discriminator, self).__init__()
        self.nc = nc
        self.ndf = ndf

        self.fc0 = nn.Sequential(
            # input is (nc) x 128 x 128
            nn.utils.spectral_norm(nn.Conv2d(nc, ndf, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True)
        )

        self.fc1 = nn.Sequential(
            # state size. (ndf) x 64 x 64
            nn.utils.spectral_norm(nn.Conv2d(ndf + nc, ndf * 2, 4, 2, 1, bias=False)),
            # "+ nc" because of multi scale gradient
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ndf * 2)
        )

        self.fc2 = nn.Sequential(
            # state size. (ndf*2) x 32 x 32
            nn.utils.spectral_norm(nn.Conv2d(ndf * 2 + nc, ndf * 4, 4, 2, 1, bias=False)),
            # "+ nc" because of multi scale gradient
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ndf * 4)
        )

        self.fc3 = nn.Sequential(
            # state size. (ndf*4) x 16 x 16e
            nn.utils.spectral_norm(nn.Conv2d(ndf * 4 + nc, ndf * 8, 4, 2, 1, bias=False)),
            # "+ nc" because of multi scale gradient
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ndf * 8),
        )

        self.fc4 = nn.Sequential(
            # state size. (ndf*8) x 8 x 8
            nn.utils.spectral_norm(nn.Conv2d(ndf * 8 + nc, ndf * 16, 4, 2, 1, bias=False)),
            nn.LeakyReLU(0.2, inplace=True),
            nn.BatchNorm2d(ndf * 16)
        )

        self.fc5 = nn.Sequential(
            # state size. (ndf*8) x 4 x 4
            nn.utils.spectral_norm(nn.Conv2d(ndf * 16 + nc, 1, 4, 1, 0, bias=False)),
            nn.Sigmoid()
        )

        # state size. 1 x 1 x 1

    def forward(self, input, detach_or_not):
        # When we train i ncombination with generator we use multi scale gradient.
        x, x_0_r, x_1_r, x_2_r, x_3_r, x_4_r = input
        if detach_or_not:
            x = x.detach()

        x_0 = self.fc0(x)

        x_0 = torch.cat((x_0, x_4_r), dim=1)  # Concat Multi-Scale Gradient
        x_1 = self.fc1(x_0)

        x_1 = torch.cat((x_1, x_3_r), dim=1)  # Concat Multi-Scale Gradient
        x_2 = self.fc2(x_1)

        x_2 = torch.cat((x_2, x_2_r), dim=1)  # Concat Multi-Scale Gradient
        x_3 = self.fc3(x_2)

        x_3 = torch.cat((x_3, x_1_r), dim=1)  # Concat Multi-Scale Gradient
        x_4 = self.fc4(x_3)

        x_4 = torch.cat((x_4, x_0_r), dim=1)  # Concat Multi-Scale Gradient
        x_5 = self.fc5(x_4)

        return x_5

class DCGAN(pl.LightningModule):

    def __init__(self, hparams, checkpoint_folder, experiment_name):
        super().__init__()
        self.hparams = hparams
        self.checkpoint_folder = checkpoint_folder
        self.experiment_name = experiment_name

        # networks
        self.generator = Generator(ngf=hparams.ngf, nc=hparams.nc, latent_dim=hparams.latent_dim)
        self.discriminator = Discriminator(ndf=hparams.ndf, nc=hparams.nc)
        self.generator.apply(weights_init)
        self.discriminator.apply(weights_init)

        # cache for generated images
        self.generated_imgs = None
        self.last_imgs = None

        # For experience replay
        self.exp_replay_dis = torch.tensor([])

    def forward(self, z):
        return self.generator(z)

    def adversarial_loss(self, y_hat, y):
        return F.binary_cross_entropy(y_hat, y)

    def training_step(self, batch, batch_nb, optimizer_idx):
        # For adding Instance noise for more visit: https://www.inference.vc/instance-noise-a-trick-for-stabilising-gan-training/
        std_gaussian = max(0, self.hparams.level_of_noise - (
                (self.hparams.level_of_noise * 2) * (self.current_epoch / self.hparams.epochs)))
        AddGaussianNoiseInst = AddGaussianNoise(std=std_gaussian)  # the noise decays over time

        imgs, _ = batch
        imgs = AddGaussianNoiseInst(imgs)  # Adding instance noise to real images
        self.last_imgs = imgs

        # train generator
        if optimizer_idx == 0:
            # sample noise
            z = torch.randn(imgs.shape[0], self.hparams.latent_dim, 1, 1).cuda()

            # generate images
            self.generated_imgs = self(z)

            # ground truth result (ie: all fake)
            g_loss = self.adversarial_loss(self.discriminator(self.generated_imgs, False), get_valid_labels(self.generated_imgs[0]))  # adversarial loss is binary cross-entropy; [0] is the image of the last layer

            tqdm_dict = {'g_loss': g_loss}
            log = {'g_loss': g_loss, "std_gaussian": std_gaussian}
            output = OrderedDict({
                'loss': g_loss,
                'progress_bar': tqdm_dict,
                'log': log
            })
            return output

        # train discriminator
        if optimizer_idx == 1:
            # Measure discriminator's ability to classify real from generated samples
            # how well can it label as real?
            real_loss = self.adversarial_loss(
                self.discriminator([imgs, resize2d(imgs, 4), resize2d(imgs, 8), resize2d(imgs, 16), resize2d(imgs, 32), resize2d(imgs, 64)],
                                   False), get_valid_labels(imgs))

            fake_loss = self.adversarial_loss(self.discriminator(self.generated_imgs, True), get_unvalid_labels(
                self.generated_imgs[0]))  # how well can it label as fake?; [0] is the image of the last layer

            # discriminator loss is the average of these
            d_loss = (real_loss + fake_loss) / 2

            tqdm_dict = {'d_loss': d_loss}
            log = {'d_loss': d_loss, "std_gaussian": std_gaussian}
            output = OrderedDict({
                'loss': d_loss,
                'progress_bar': tqdm_dict,
                'log': log
            })
            return output

    def configure_optimizers(self):
        lr_gen = self.hparams.lr_gen
        lr_dis = self.hparams.lr_dis
        b1 = self.hparams.b1
        b2 = self.hparams.b2

        opt_g = torch.optim.Adam(self.generator.parameters(), lr=lr_gen, betas=(b1, b2))
        opt_d = torch.optim.Adam(self.discriminator.parameters(), lr=lr_dis, betas=(b1, b2))
        return [opt_g, opt_d], []

    def backward(self, trainer, loss, optimizer, optimizer_idx: int) -> None:
        loss.backward(retain_graph=True)

    def train_dataloader(self):
        # transform = transforms.Compose([transforms.Resize((self.hparams.image_size, self.hparams.image_size)),
        #                                 transforms.ToTensor(),
        #                                 transforms.Normalize([0.5], [0.5])])
        # dataset = torchvision.datasets.MNIST(os.getcwd(), train=False, download=True, transform=transform)
        # return DataLoader(dataset, batch_size=self.hparams.batch_size)
        # transform = transforms.Compose([transforms.Resize((self.hparams.image_size, self.hparams.image_size)),
        #                                 transforms.ToTensor(),
        #                                 transforms.Normalize([0.5], [0.5])
        #                                 ])

        # train_dataset = torchvision.datasets.ImageFolder(
        #     root="./drive/My Drive/datasets/flower_dataset/",
        #     # root="./drive/My Drive/datasets/ghibli_dataset_small_overfit/",
        #     transform=transform
        # )
        # return DataLoader(train_dataset, num_workers=self.hparams.num_workers, shuffle=True,
        #                   batch_size=self.hparams.batch_size)

        transform = transforms.Compose([transforms.Resize((self.hparams.image_size, self.hparams.image_size)),
                                        transforms.ToTensor(),
                                        transforms.Normalize([0.5], [0.5])
                                        ])
        train_dataset = torchvision.datasets.ImageFolder(
            root="ghibli_dataset_small_overfit/",
            transform=transform
        )
        return DataLoader(train_dataset, num_workers=self.hparams.num_workers, shuffle=True,
                          batch_size=self.hparams.batch_size)

    def on_epoch_end(self):
        z = torch.randn(4, self.hparams.latent_dim, 1, 1).cuda()
        # match gpu device (or keep as cpu)
        if self.on_gpu:
            z = z.cuda(self.last_imgs.device.index)

        # log sampled images
        sample_imgs = self.generator(z)[0]
        torchvision.utils.save_image(sample_imgs, f'generated_images_epoch{self.current_epoch}.png')

        # save model
        if self.current_epoch % self.hparams.save_model_every_epoch == 0:
            trainer.save_checkpoint(
                self.checkpoint_folder + "/" + self.experiment_name + "_epoch_" + str(self.current_epoch) + ".ckpt")

from argparse import Namespace

args = {
    'batch_size': 128, # batch size
    'lr_gen': 0.0003,  # TTUR;learnin rate of both networks; tested value: 0.0002
    'lr_dis': 0.0003,  # TTUR;learnin rate of both networks; tested value: 0.0002
    'b1': 0.5,  # Momentum for adam; tested value(dcgan paper): 0.5
    'b2': 0.999,  # Momentum for adam; tested value(dcgan paper): 0.999
    'latent_dim': 256,  # tested value which worked(in V4_1): 100
    'nc': 3,  # number of color channels
    'ndf': 8,  # number of discriminator features
    'ngf': 8,  # number of generator features
    'epochs': 4,  # the maxima lamount of epochs the algorith should run
    'save_model_every_epoch': 1,  # how often we save our model
    'image_size': 128, # size of the image
    'num_workers': 3,
    'level_of_noise': 0.1,  # how much instance noise we introduce(std; tested value: 0.15 and 0.1
    'experience_save_per_batch': 1,  # this value should be very low; tested value which works: 1
    'experience_batch_size': 50  # this value shouldnt be too high; tested value which works: 50
}
hparams = Namespace(**args)

# Parameters
experiment_name = "DCGAN_6_2_MNIST_128px"
dataset_name = "mnist"
checkpoint_folder = "DCGAN/"
tags = ["DCGAN", "128x128"]
dirpath = Path(checkpoint_folder)

# defining net
net = DCGAN(hparams, checkpoint_folder, experiment_name)

torch.autograd.set_detect_anomaly(True)
trainer = pl.Trainer( # resume_from_checkpoint="DCGAN_V4_2_GHIBLI_epoch_999.ckpt",
    max_epochs=args["epochs"],
    gpus=1
)

trainer.fit(net)

``


Instruction: Pytorch tensor.save() produces huge files for small tensors from MNIST

Input:
I'm working with MNIST dataset from Kaggle challange and have troubles preprocessing with data. Furthermore, I don't know what are the best practices and was wondering if you could advise me on that.

Disclaimer: I can't just use torchvision.datasets.mnist because I need to use Kaggle's data for training and submission.

In this tutorial, it was advised to create a Dataset object loading .pt tensors from files, to fully utilize GPU. In order to achieve that, I needed to load the csv data provided by Kaggle and save it as .pt files:

import pandas as pd
import torch
import numpy as np

# import data
digits_train = pd.read_csv('data/train.csv')

train_tensor = torch.tensor(digits_train.drop(label, axis=1).to_numpy(), dtype=torch.int)
labels_tensor = torch.tensor(digits_train[label].to_numpy())

for i in range(train_tensor.shape[0]):
    torch.save(train_tensor[i], "data/train-" + str(i) + ".pt")

Each train_tensor[i].shape is torch.Size([1, 784])

However, each such .pt file has size of about 130MB.
A tensor of the same size, with randomly generated integers, has size of 6.6kB.
Why are these tensors so huge, and how can I reduce their size?

Dataset is 42 000 samples. Should I even bother with batching this data? Should I bother with saving tensors to separate files, rather than loading them all into RAM and then slicing into batches? What is the most optimal approach here?

Output:
As explained in this discussion, torch.save() saves the whole tensor, not just the slice. You need to explicitly copy the data using clone().

Don't worry, at runtime the data is only allocated once unless you explicitly create copies.

As a general advice: If the data easily fits into your memory, just load it at once. For MNIST with 130 MB that's certainly the case.

However, I would still batch the data because it converges faster. Look up the advantages of SGD for more details.


Instruction: How can I do return_sequences for a stacked LSTM model with PyTorch?

Input:
I have a Tensorflow / Keras model with:

        self.model.add(Bidirectional(LSTM(lstm1_size, input_shape=(
            seq_length, feature_dim), return_sequences=True)))
        self.model.add(BatchNormalization())
        self.model.add(Dropout(0.2))

        self.model.add(Bidirectional(
            LSTM(lstm2_size, return_sequences=True)))
        self.model.add(BatchNormalization())
        self.model.add(Dropout(0.2))

        # BOTTLENECK HERE

        self.model.add(Bidirectional(
            LSTM(lstm3_size, return_sequences=True)))
        self.model.add(BatchNormalization())
        self.model.add(Dropout(0.2))

        self.model.add(Bidirectional(
            LSTM(lstm4_size, return_sequences=True)))
        self.model.add(BatchNormalization())
        self.model.add(Dropout(0.2))

        self.model.add(Bidirectional(
            LSTM(lstm5_size, return_sequences=True)))
        self.model.add(BatchNormalization())
        self.model.add(Dropout(0.2))

        self.model.add(Dense(feature_dim, activation='linear'))

How do I create a stacked PyTorch model with the return_sequences? My understanding of return_sequences is that it returns the "output" of each layer of LSTM's which are then fed into the next layer.

How would I accomplish this with PyToch?

Output:
PyTorch always returns sequences.

https://pytorch.org/docs/stable/nn.html#lstm

Example:

import torch as t

batch_size = 2
time_steps = 10
features = 2
data = t.empty(batch_size, time_steps, features).normal_()

lstm = t.nn.LSTM(input_size=2, hidden_size=3, bidirectional=True, batch_first=True)

output, (h_n, c_n) = lstm(data)
[output.shape, h_n.shape, c_n.shape]

  [torch.Size([2, 10, 6]), torch.Size([2, 2, 3]), torch.Size([2, 2, 3])]

class Net(t.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.lstm_1 = t.nn.LSTM(input_size=2, hidden_size=3, bidirectional=True, batch_first=True)
        self.lstm_2 = t.nn.LSTM(input_size=2*3, hidden_size=4, bidirectional=True, batch_first=True)

    def forward(self, input):
        output, (h_n, c_n) = self.lstm_1(input)
        output, (h_n, c_n) = self.lstm_2(output)
        return output

net = Net()

net(data).shape

  torch.Size([2, 10, 8])


Instruction: Pytorch Custom Module Using Existing CNN Module

Input:
I want to access and edit individual modules within a torchvision module and adjust the input.

I know you can edit submodules like this:

import torchvision
resnet18 = torchvision.models.resnet18()
print(resnet18._modules['conv1'])
# Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)

But I want to create a custom Net(nn.Module) kind of class so I can add additional layers later:

class Sonar(resnet18):
    pass

Throws the error:

----> 1 class Sonar(resnet18):
      2     pass

/usr/local/lib/python3.7/dist-packages/torchvision/models/resnet.py in __init__(self, block, layers, num_classes, zero_init_residual, groups, width_per_group, replace_stride_with_dilation, norm_layer)
    142         self.relu = nn.ReLU(inplace=True)
    143         self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
--> 144         self.layer1 = self._make_layer(block, 64, layers[0])
    145         self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
    146                                        dilate=replace_stride_with_dilation[0])

/usr/local/lib/python3.7/dist-packages/torchvision/models/resnet.py in _make_layer(self, block, planes, blocks, stride, dilate)
    176             self.dilation *= stride
    177             stride = 1
--> 178         if stride != 1 or self.inplanes != planes * block.expansion:
    179             downsample = nn.Sequential(
    180                 conv1x1(self.inplanes, planes * block.expansion, stride),

AttributeError: 'str' object has no attribute 'expansion'

Trying again with AlexNet

alexnet = torchvision.models.AlexNet()
class Sonar(alexnet):
    pass

Throws the error:

      1 alexnet = torchvision.models.AlexNet()
----> 2 class Sonar(alexnet):
      3     pass

TypeError: __init__() takes from 1 to 2 positional arguments but 4 were given

Output:
The following should work well:

import torch
import torchvision

class Sonar(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.ins = torchvision.models.resnet18(pretrained=True)
        self.fc1 = torch.nn.Linear(1000, 1) #adding layers
    def forward(self, x):
        out = self.ins(x)
        out = self.fc1(out)
        return out

def run():
    return Sonar()

net = run()
print(net(torch.ones(1,3,224,224))) #testing


Instruction: Torchscript incompatible with torch.cat for tensor lists

Input:
Torch.cat throws error for tensor lists when used within torchscript

Here is a minimum reproducable example to reproduce the error

import torch
import torch.nn as nn

"""
Smallest working bug for torch.cat torchscript
"""

class Model(nn.Module):
    """dummy model for showing error"""

    def __init__(self):
        super(Model, self).__init__()
        pass

    def forward(self):
        a = torch.rand([6, 1, 12])
        b = torch.rand([6, 1, 12])
        out = torch.cat([a, b], axis=2)
        return out

if __name__ == '__main__':
    model = Model()
    print(model())  # works
    torch.jit.script(model)  # throws error

The expected result would be a torchscript output for torch.cat. Here is the error message provided:

File "/home/anil/.conda/envs/rnn/lib/python3.7/site-packages/torch/jit/__init__.py", line 1423, in _create_methods_from_stubs
    self._c._create_methods(self, defs, rcbs, defaults)
RuntimeError:
Arguments for call are not valid.
The following operator variants are available:

  aten::cat(Tensor[] tensors, int dim=0) -> (Tensor):
  Keyword argument axis unknown.

  aten::cat.out(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> (Tensor(a!)):
  Argument out not provided.

The original call is:
at smallest_working_bug_torch_cat_torchscript.py:19:14
    def forward(self):
        a = torch.rand([6, 1, 12])
        b = torch.rand([6, 1, 12])
        out = torch.cat([a, b], axis=2)
              ~~~~~~~~~ <--- HERE
        return out

Kindly let me know of a fix or a workaround for this problem.

Thanks!

Output:
changing axis to dim fixes the error,
Original solution was posted here


Instruction: Beginner PyTorch - RuntimeError: shape '[16, 400]' is invalid for input of size 9600

Input:
I'm trying to build a CNN but I get this error:

---> 52         x = x.view(x.size(0), 5 * 5 * 16)
RuntimeError: shape '[16, 400]' is invalid for input of size 9600

It's not clear for me what the inputs of the 'x.view' line should be. Also, I don't really understand how many times I should have this 'x.view' function in my code. Is it only once, after the 3 convolutional layers and 2 linear layers? Or is it 5 times, one after every layer?

Here's my code:

CNN

import torch.nn.functional as F

# Convolutional neural network
class ConvNet(nn.Module):

    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()

        self.conv1 = nn.Conv2d(
            in_channels=3,
            out_channels=16,
            kernel_size=3)

        self.conv2 = nn.Conv2d(
            in_channels=16,
            out_channels=24,
            kernel_size=4)

        self.conv3 = nn.Conv2d(
            in_channels=24,
            out_channels=32,
            kernel_size=4)

        self.dropout = nn.Dropout2d(p=0.3)

        self.pool = nn.MaxPool2d(2)

        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(512, 10)

        self.final = nn.Softmax(dim=1)

    def forward(self, x):

        print('shape 0 ' + str(x.shape))

        x = F.max_pool2d(F.relu(self.conv1(x)), 2)
        x = self.dropout(x)

        print('shape 1 ' + str(x.shape))

        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = self.dropout(x)

        print('shape 2 ' + str(x.shape))

        # x = F.max_pool2d(F.relu(self.conv3(x)), 2)
        # x = self.dropout(x)

        x = F.interpolate(x, size=(5, 5))
        x = x.view(x.size(0), 5 * 5 * 16)

        x = self.fc1(x)

        return x

net = ConvNet()

Can someone help me understand the problem?

The output of 'x.shape' is:

shape 0 torch.Size([16, 3, 256, 256])

shape 1 torch.Size([16, 16, 127, 127])

shape 2 torch.Size([16, 24, 62, 62])

Thanks

Output:
This means that instead the product of the channel and spatial dimensions is not 5*5*16. To flatten the tensor, replace x = x.view(x.size(0), 5 * 5 * 16) with:

x = x.view(x.size(0), -1)

And self.fc1 = nn.Linear(600, 120) with:

self.fc1 = nn.Linear(600, 120)


Instruction: RuntimeError: expected scalar type Long but found Float

Input:
I can't get the dtypes to match, either the loss wants long or the model wants float if I change my tensors to long. The shape of the tensors are 42000, 1, 28, 28 and 42000. I'm not sure where I can change what dtypes are required for the model or loss.

I'm not sure if dataloader is required, using Variable didn't work either.

dataloaders_train = torch.utils.data.DataLoader(Xt_train, batch_size=64)

dataloaders_test = torch.utils.data.DataLoader(Yt_train, batch_size=64)

class Network(nn.Module):
    def __init__(self):
        super().__init__()

        self.hidden = nn.Linear(42000, 256)

        self.output = nn.Linear(256, 10)

        self.sigmoid = nn.Sigmoid()
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):

        x = self.hidden(x)
        x = self.sigmoid(x)
        x = self.output(x)
        x = self.softmax(x)

        return x

model = Network()

input_size = 784
hidden_sizes = [28, 64]
output_size = 10
model = nn.Sequential(nn.Linear(input_size, hidden_sizes[0]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[0], hidden_sizes[1]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[1], output_size),
                      nn.Softmax(dim=1))
print(model)

criterion = nn.NLLLoss()
optimizer = optim.SGD(model.parameters(), lr=0.003)

epochs = 5

for e in range(epochs):
    running_loss = 0
    for images, labels in zip(dataloaders_train, dataloaders_test):

        images = images.view(images.shape[0], -1)
        #images, labels = Variable(images), Variable(labels)
        print(images.dtype)
        print(labels.dtype)

        optimizer.zero_grad()

        output = model(images)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
    else:
        print(f"Training loss: {running_loss}")

Which gives

RuntimeError                              Traceback (most recent call last)
<ipython-input-128-68109c274f8f> in <module>
     11
     12         output = model(images)
---> 13         loss = criterion(output, labels)
     14         loss.backward()
     15         optimizer.step()

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    530             result = self._slow_forward(*input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():
    534             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    202
    203     def forward(self, input, target):
--> 204         return F.nll_loss(input, target, weight=self.weight, ignore_index=self.ignore_index, reduction=self.reduction)
    205
    206

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1836                          .format(input.size(0), target.size(0)))
   1837     if dim == 2:
-> 1838         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   1839     elif dim == 4:
   1840         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: expected scalar type Long but found Float

Output:
LongTensor is synonymous with integer. PyTorch won't accept a FloatTensor as categorical target, so it's telling you to cast your tensor to LongTensor. This is how you should change your target dtype:

Yt_train = Yt_train.type(torch.LongTensor)

This is very well documented on the PyTorch website, you definitely won't regret spending a minute or two reading this page. PyTorch essentially defines nine CPU tensor types and nine GPU tensor types:

╔══════════════════════════╦═══════════════════════════════╦════════════════════╦═════════════════════════╗
║        Data type         ║             dtype             ║     CPU tensor     ║       GPU tensor        ║
╠══════════════════════════╬═══════════════════════════════╬════════════════════╬═════════════════════════╣
║ 32-bit floating point    ║ torch.float32 or torch.float  ║ torch.FloatTensor  ║ torch.cuda.FloatTensor  ║
║ 64-bit floating point    ║ torch.float64 or torch.double ║ torch.DoubleTensor ║ torch.cuda.DoubleTensor ║
║ 16-bit floating point    ║ torch.float16 or torch.half   ║ torch.HalfTensor   ║ torch.cuda.HalfTensor   ║
║ 8-bit integer (unsigned) ║ torch.uint8                   ║ torch.ByteTensor   ║ torch.cuda.ByteTensor   ║
║ 8-bit integer (signed)   ║ torch.int8                    ║ torch.CharTensor   ║ torch.cuda.CharTensor   ║
║ 16-bit integer (signed)  ║ torch.int16 or torch.short    ║ torch.ShortTensor  ║ torch.cuda.ShortTensor  ║
║ 32-bit integer (signed)  ║ torch.int32 or torch.int      ║ torch.IntTensor    ║ torch.cuda.IntTensor    ║
║ 64-bit integer (signed)  ║ torch.int64 or torch.long     ║ torch.LongTensor   ║ torch.cuda.LongTensor   ║
║ Boolean                  ║ torch.bool                    ║ torch.BoolTensor   ║ torch.cuda.BoolTensor   ║
╚══════════════════════════╩═══════════════════════════════╩════════════════════╩═════════════════════════╝


Instruction: Beginner PyTorch : RuntimeError: size mismatch, m1: [16 x 2304000], m2: [600 x 120]

Input:
I'm a beginner with PyTorch and building NNs in general and I'm kinda stuck.

I have this CNN architecture:

class ConvNet(nn.Module):

    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()

        self.conv1 = nn.Conv2d(
            in_channels=3,
            out_channels=16,
            kernel_size=3)

        self.conv2 = nn.Conv2d(
            in_channels=16,
            out_channels=24,
            kernel_size=4)

        self.conv3 = nn.Conv2d(
            in_channels=24,
            out_channels=32,
            kernel_size=4)

        self.dropout = nn.Dropout2d(p=0.3)

        self.pool = nn.MaxPool2d(2)

        self.fc1 = nn.Linear(600, 120)
        self.fc2 = nn.Linear(512, 10)

        self.final = nn.Softmax(dim=1)

    def forward(self, x):

        # conv 3 layers

        x = F.max_pool2d(F.relu(self.conv1(x)), 2)  # output of conv layers
        x = self.dropout(x)

        x = F.max_pool2d(F.relu(self.conv2(x)), 2)  # output of conv layers
        x = self.dropout(x)

        x = F.max_pool2d(F.relu(self.conv3(x)), 2)  # output of conv layers
        x = self.dropout(x)

        # linear layer

        x = F.interpolate(x, size=(600, 120))
        x = x.view(x.size(0), -1)

        x = self.fc1(x)

        return x

But when I try to train with my images, it doesn't work and I have this error:

RuntimeError: size mismatch, m1: [16 x 2304000], m2: [600 x 120]

I would like to add a second linear layer (self.fc2) as well as a final SoftMax layer (self.final) but since I'm stuck at the first linear layer I cannot make any progress.

Output:
The input dimension of self.fc1 needs to match the feature (second) dimension of your flattened tensor. So instead of doing self.fc1 = nn.Linear(600, 120), you can replace this with self.fc1 = nn.Linear(2304000, 120).

Keep in mind that because you are using fully-connected layers, the model cannot be input size invariant (unlike Fully-Convolutional Networks). If you change the size of the channel or spatial dimensions before x = x.view(x.size(0), -1) (like you did moving from the last question to this one), the input dimension of self.fc1 will have to change accordingly.


Instruction: Why is the true positive - false negative distribution always the same

Input:
I have a neural network that I use it for binary classification. I change the size of training data and predict on the test set. By looking at the results, the difference between tp and fn is always the same and the difference between tn and fp is always the same. For example, in iteration #2, tp#2 - tp#1 = -91 and fn#2 - fn#1 = +91. Also, fp#2 - fp#1 = -46 and tn#2 - tn#1 = +46. As another example, tp#3 - tp#2 = -35 and fn#2 - fn#2 = +35.

Iteration #1
tn=119, fp=173, fn=110, tp=407
Iteration #2
tn=165, fp=127, fn=201, tp=316
Iteration #3
tn=176, fp=116, fn=236, tp=281
Iteration #4
tn=157, fp=135, fn=207, tp=310
Iteration #5
tn=155, fp=137, fn=214, tp=303

I have tried various architectures of neural nets, but I always get the same numbers. Do you have an idea what is wrong.

The following is a very simple network that I use:

class AllCnns(nn.Module):
    def __init__(self, vocab_size, embedding_size):
        torch.manual_seed(0)
        super(AllCnns, self).__init__()
        self.word_embeddings = nn.Embedding(vocab_size, embedding_size)
        self.conv1 = nn.Conv1d(embedding_size, 64, 3)
        self.drop1 = nn.Dropout(0.3)
        self.max_pool1 = nn.MaxPool1d(2)
        self.flat1 = nn.Flatten()
        self.fc1 = nn.Linear(64*80, 100)
        self.fc2 = nn.Linear(100, 1)

    def forward(self, sentence):
        embedding = self.word_embeddings(sentence).permute(0, 2, 1)
        conv1 = F.relu(self.conv1(embedding))
        drop1 = self.drop1(conv1)
        max_pool1 = self.max_pool1(drop1)
        flat1 = self.flat1(max_pool1)
        fc1 = F.relu(self.fc1(flat1))
        fc2 = torch.sigmoid(self.fc2(fc1))
        return fc2

Output:
I think it should be the same.
The sum of tn(true negative) and fp(false positive) adds up to the total 'real' negative values, and same goes for the other two.
So as long as you are using the same data,

tn + fp = 292(total negative values)
fn + tp = 517(total positive values)

these equations are always true.
So tn#1 + fp#1 = tn#2 + fp#2 so tn#1 - tn#2 = fp#2 - fp#1


Instruction: Pytorch copy a neuron in a layer

Input:
I am using pytorch 0.3.0. I'm trying to selectively copy a neuron and it's weights within the same layer, then replace the original neuron with an another set of weights. Here's my attempt at that:

reshaped_data2 = data2.unsqueeze(0)
new_layer_data = torch.cat([new_layer.data, reshaped_data2], dim=0)
new_layer_data[i] = data1
new_layer.data.copy_(new_layer_data)

First I unsqueezed data2 to make it a 1*X tensor instead of 0*X.
Then I concatenate my layer's tensor with the reshaped data2 along dimension 0.
I then replace the original data2 located at index i with data1.
Finally, I copy all of that into my layer.

The error I get is:

RuntimeError: inconsistent tensor size, expected tensor [10 x 128] and src [11 x 128] to have the same number of elements, but got 1280 and 1408 elements respectively at /Users/soumith/code/builder/wheel/pytorch-src/torch/lib/TH/generic/THTensorCopy.c:86

If I do a simple assignment instead of copy I get

RuntimeError: The expanded size of the tensor (11) must match the existing size (10) at non-singleton dimension 1. at /Users/soumith/code/builder/wheel/pytorch-src/torch/lib/TH/generic/THTensor.c:309

I understand the error, but what is the right way to go about this?

Output: The solution here is to create a new model with the correct size and pass in weights as default values. No dynamic expansion solution was found.


Instruction: Using past and attention_mask at the same time for gpt2

Input:
I am processing a batch of sentences with different lengths, so I am planning to take advantage of the padding + attention_mask functionality in gpt2 for that.

At the same time, for each sentence I need to add a suffix phrase and run N different inferences. For instance, given the sentence "I like to drink coke", I may need to run two different inferences: "I like to drink coke. Coke is good" and "I like to drink coke. Drink is good". Thus, I am trying to improve the inference time for this by using the "past" functionality: https://huggingface.co/transformers/quickstart.html#using-the-past so I just process the original sentence (e.g. "I like to drink coke") once, and then I somehow expand the result to be able to be used with two other sentences: "Coke is good" and "Drink is good".

Below you will find a simple code that is trying to represent how I was trying to do this. For simplicity I'm just adding a single suffix phrase per sentence (...but I still hope my original idea is possible though):

from transformers.tokenization_gpt2 import GPT2Tokenizer
from transformers.modeling_gpt2 import GPT2LMHeadModel

tokenizer = GPT2Tokenizer.from_pretrained('gpt2', pad_token='<|endoftext|>')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# Complete phrases are: "I like to drink soda without sugar" and "Go watch TV alone, I am not going"
docs = ["I like to drink soda", "Go watch TV"]
docs_tensors = tokenizer.batch_encode_plus(
    [d for d in docs], pad_to_max_length=True, return_tensors='pt')

docs_next = ["without sugar", "alone, I am not going"]
docs_next_tensors = tokenizer.batch_encode_plus(
    [d for d in docs_next], pad_to_max_length=True, return_tensors='pt')

# predicting the first part of each phrase
_, past = model(docs_tensors['input_ids'], attention_mask=docs_tensors['attention_mask'])

# predicting the rest of the phrase
logits, _ = model(docs_next_tensors['input_ids'], attention_mask=docs_next_tensors['attention_mask'], past=past)
logits = logits[:, -1]
_, top_indices_results = logits.topk(30)

The error I am getting is the following:

Traceback (most recent call last):
  File "/Applications/PyCharm CE.app/Contents/plugins/python-ce/helpers/pydev/pydevd.py", line 1434, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "/Applications/PyCharm CE.app/Contents/plugins/python-ce/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/Users/damiox/Workspace/xxLtd/yy/stress-test-withpast2.py", line 26, in <module>
    logits, _ = model(docs_next_tensors['input_ids'], attention_mask=docs_next_tensors['attention_mask'], past=past)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/transformers/modeling_gpt2.py", line 593, in forward
    inputs_embeds=inputs_embeds,
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/transformers/modeling_gpt2.py", line 476, in forward
    hidden_states, layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask[i]
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/transformers/modeling_gpt2.py", line 226, in forward
    self.ln_1(x), layer_past=layer_past, attention_mask=attention_mask, head_mask=head_mask
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/transformers/modeling_gpt2.py", line 189, in forward
    attn_outputs = self._attn(query, key, value, attention_mask, head_mask)
  File "/Users/damiox/.local/share/virtualenvs/yy-uMxmjV2h/lib/python3.7/site-packages/transformers/modeling_gpt2.py", line 150, in _attn
    w = w + attention_mask
RuntimeError: The size of tensor a (11) must match the size of tensor b (6) at non-singleton dimension 3

Process finished with exit code 1

Initially I thought this was related to https://github.com/huggingface/transformers/issues/3031 - so I re-built latest master to try the fix, but I still experience the issue.

Output:
In order to make your current code snippet work, you will have combine the previous and new attention mask as follows:

from transformers.tokenization_gpt2 import GPT2Tokenizer
from transformers.modeling_gpt2 import GPT2LMHeadModel
import torch

tokenizer = GPT2Tokenizer.from_pretrained('gpt2', pad_token='<|endoftext|>')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# Complete phrases are: "I like to drink soda without sugar" and "Go watch TV alone, I am not going"
docs = ["I like to drink soda", "Go watch TV"]
docs_tensors = tokenizer.batch_encode_plus(
    [d for d in docs], pad_to_max_length=True, return_tensors='pt')

docs_next = ["without sugar", "alone, I am not going"]
docs_next_tensors = tokenizer.batch_encode_plus(
    [d for d in docs_next], pad_to_max_length=True, return_tensors='pt')

# predicting the first part of each phrase
_, past = model(docs_tensors['input_ids'], attention_mask=docs_tensors['attention_mask'])

# predicting the rest of the phrase
attn_mask = torch.cat([docs_tensors['attention_mask'], docs_next_tensors['attention_mask']], dim=-1)
logits, _ = model(docs_next_tensors['input_ids'], attention_mask=attn_mask, past=past)
logits = logits[:, -1]
_, top_indices_results = logits.topk(30)

For the case that you want to test two possible suffixes for a sentence start you probably will have to clone your past variable as many times as you have suffixes. That means that the batch size of your prefix input_ids has to match the batch size of your suffix input_ids in order to make it work.

Also you have to change the positional encodings input of your suffix input_ids (GPT2 uses absolute positional encodings) if one of your prefix input_ids is padded (this is not shown in the code above - please take a look at https://github.com/huggingface/transformers/issues/3021 to see how it's done).


Instruction: How to convert an list of image into Pytorch Tensor

Input:
I have a list called wordImages. It contains images in np.array format with different width & height.

How Do I convert this into a tensor and use this instead of my_dataset in the below code?

Currently i am using this. But I need to save/read images

demo_data = RawDataset(root="output_craft/", opt=opt)

demo_loader = torch.utils.data.DataLoader(
                demo_data , batch_size=opt.batch_size,
                shuffle=False,
                num_workers=int(opt.workers),
                collate_fn=AlignCollate_demo, pin_memory=True)

Output:
You can use transforms from the torchvision library to do so. You can pass whatever transformation(s) you declare as an argument into whatever class you use to create my_dataset, like so:

from torchvision import transforms as transforms

class MyDataset(data.Dataset):

    def __init__(self, transform=transforms.ToTensor()):
        self.transform = transform
        ...
    def __getitem__(self, idx):
        ...
        img_tensor = self.transform(img)
        return (img_tensor, label)


Instruction: PyTorch: Trying to backward through the graph a second time, but the buffers have already been freed

Input:
My model is:

class BaselineModel(nn.Module):
    def __init__(self, feature_dim=5, hidden_size=5, num_layers=2, batch_size=32):
        super(BaselineModel, self).__init__()
        self.num_layers = num_layers
        self.hidden_size = hidden_size

        self.lstm = nn.LSTM(input_size=feature_dim,
                            hidden_size=hidden_size, num_layers=num_layers)

    def forward(self, x, hidden):
        lstm_out, hidden = self.lstm(x, hidden)
        return lstm_out, hidden

    def init_hidden(self, batch_size):
        hidden = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        cell = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        return (hidden, cell)

My training loop looks like:

    for epoch in range(250):
        hidden = model.init_hidden(13)
        # hidden = (torch.zeros(2, 13, 5),
        #           torch.zeros(2, 13, 5))
        # model.hidden = hidden
        for i, data in enumerate(train_loader):
            inputs = data[0]
            outputs = data[1]

            print('inputs',  inputs.size())
            # print('outputs', outputs.size())

            optimizer.zero_grad()
            model.zero_grad()

            # print('inputs', inputs)
            pred, hidden = model(inputs, hidden)

            loss = loss_fn(pred[0], outputs)

            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
            optimizer.step()

I appear to get through the first epoch, then see this error: RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.

I've seen postings about it, but in my case, I am reinitializing my hidden for each batch.

Output: model.init_hidden(13) must be in the batch loop, rather than the epoch loop


Instruction: Pytorch is Throwing out of bounds Error? Expects a scalar

Input:
This is the code below, Not sure what the error being thrown is. Please can someone explain what is wrong and the fix. I'm new to pytorch and decided to try learn it using the house prices data set but ran into this error.

It is apprently something to do with scalar value or somethings but not sure that the problem is because the y value given is a scalar not a vector.

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import TensorDataset, DataLoader
import torch.optim as optim
import pandas as pd
import numpy as np
df = pd.read_csv('housepricedata.csv')
dataset = df.values
X = dataset[:,0:10]
y = dataset[:, 10]
from sklearn import preprocessing
min_max = preprocessing.MinMaxScaler()
x_scale = min_max.fit_transform(X)
y_scale = min_max.fit_transform(y_scale)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x_scale, y, test_size=0.3)

X_train = torch.FloatTensor(X_train)
X_test = torch.FloatTensor(X_test)
y_train = torch.LongTensor(y_train)
y_test = torch.LongTensor(y_test)

trainD = TensorDataset(X_train, y_train)
testD = TensorDataset(X_test, y_test)
class Model(nn.Module):
    def __init__(self, inp1=10, out=1):
        super().__init__()
        self.Dense1 = nn.Linear(inp1, 32)
        self.Dense2 = nn.Linear(32, 32)
        self.out = nn.Linear(32, out)

    def forward(self, x):
        x = F.relu(self.Dense1(x))
        x = F.relu(self.Dense2(x))
        x = self.out(x)
        return x
model = Model()
trainloader = DataLoader(trainD, batch_size=64, shuffle=False)
testloader  = DataLoader(testloader, batch_size=64, shuffle=False)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
epochs1 = 500
losses = []

epochs1 = 500
losses = []
for i in range(epochs1):
    for data in trainloader:
        X, y = data
        optimizer.zero_grad()
        output = model(X)
        loss = criterion(output, y)
        losses.append(loss)
        loss.backward()
        optimizer.step()

Error thrown:

IndexError                                Traceback (most recent call last)
 in
      5     i =+1
      6     y_pred = model.forward(X_train)
----> 7     loss = criterion(y_pred, y_train)
      8     losses.append(loss)
      9

~\Anaconda3\envs\ml1\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    530             result = self._slow_forward(*input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():
    534             hook_result = hook(self, input, result)

~\Anaconda3\envs\ml1\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
    914     def forward(self, input, target):
    915         return F.cross_entropy(input, target, weight=self.weight,
--> 916                                ignore_index=self.ignore_index, reduction=self.reduction)
    917
    918

~\Anaconda3\envs\ml1\lib\site-packages\torch\nn\functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   2019     if size_average is not None or reduce is not None:
   2020         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2021     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   2022
   2023

~\Anaconda3\envs\ml1\lib\site-packages\torch\nn\functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   1836                          .format(input.size(0), target.size(0)))
   1837     if dim == 2:
-> 1838         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   1839     elif dim == 4:
   1840         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

IndexError: Target 1 is out of bounds.

Output:
You are trying to predict a discrete class with a regression network.

When trying to predict discrete classes, one usually outputs a vector of class-probabilities - the probability for each class given the input. On the other hand, there are regression tasks in which one wants to compute a continuous function of the given input. In the regression case, the new usually outputs only one scalar value per input.

In your code you are mixing the two: on the one hand your network has a single scalar output (self.out of your model has out_features=1). On the other hand, you are using nn.CrossEntropyLoss() which is a loss for classification that expects a vector of class probabilities.


Instruction: Confusing results of transfer learning accuracy

Input:
I have just started an image classification project according to the tutorial from the documentation on the pytorch website(this).In the tutorial, there is a part of code like this:

model_ft = models.resnet50(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 20)

And I have known the reason why the fc layer is supposed to be changed .Since my project needs to classify 20 classes , so I just changed the parameters from 2 to 20. But , I just get the accuracy of around 60%. When I dont change the fc layer like this:

model_ft = se_resnet50(pretrained = True)

It turned out that the accuracy reaches 93.75% which is much better compared with the former results.

I just couldnt figure out why I get worse classification results when I modify the fc layer. Shouldn't it be modified?

Output:
It's probably more difficult for the network to find the matching class between 20 classes than between two classes.

For example if you give it a dog image and it need to classify it between cat, dog and horse it could send 60% cat, 30% dog 10% horse and then be wrong
while if it needs to classify it only between dog and horse it would give may be 75% dog, 25% horse and then be wright.

The finetunnig will also be longer so you could have better result if you train it longer with the 20 classes if you haven't stop it after convergence but after a fix number of epochs.


Instruction: PyTorch LSTM has nan for MSELoss

Input:
My model is:

class BaselineModel(nn.Module):
    def __init__(self, feature_dim=5, hidden_size=5, num_layers=2, batch_size=32):
        super(BaselineModel, self).__init__()
        self.num_layers = num_layers
        self.hidden_size = hidden_size

        self.lstm = nn.LSTM(input_size=feature_dim,
                            hidden_size=hidden_size, num_layers=num_layers)

    def forward(self, x, hidden):
        lstm_out, hidden = self.lstm(x, hidden)
        return lstm_out, hidden

    def init_hidden(self, batch_size):
        hidden = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        cell = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        return (hidden, cell)

Training looks like:

train_loader = torch.utils.data.DataLoader(
    train_set, batch_size=BATCH_SIZE, shuffle=True, **params)

model = BaselineModel(batch_size=BATCH_SIZE)
optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001)
loss_fn = torch.nn.MSELoss(reduction='sum')

for epoch in range(250):

    # hidden = (torch.zeros(2, 13, 5),
    #           torch.zeros(2, 13, 5))
    # model.hidden = hidden
    for i, data in enumerate(train_loader):
        hidden = model.init_hidden(13)
        inputs = data[0]
        outputs = data[1]

        print('inputs',  inputs.size())
        # print('outputs', outputs.size())

        # optimizer.zero_grad()
        model.zero_grad()

        # print('inputs', inputs)
        pred, hidden = model(inputs, hidden)

        loss = loss_fn(pred, outputs)

        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()

        print('Epoch: ', epoch, '\ti: ', i, '\tLoss: ', loss)

I have gradient clipping set already, which seems to be the recommended solution. But after even the first step, I get:

  Epoch:  0       i:  0   Loss:  tensor(nan, grad_fn=)

Output:
I suspect your issue has to do with your outputs / data[1] (it would help if you show examples of your train_set). Running the following piece of code gives no nan, but I forced shape of output by hand before calling the loss_fn(pred, outputs) :

class BaselineModel(nn.Module):
    def __init__(self, feature_dim=5, hidden_size=5, num_layers=2, batch_size=32):
        super(BaselineModel, self).__init__()
        self.num_layers = num_layers
        self.hidden_size = hidden_size

        self.lstm = nn.LSTM(input_size=feature_dim,
                            hidden_size=hidden_size, num_layers=num_layers)

    def forward(self, x, hidden):
        lstm_out, hidden = self.lstm(x, hidden)
        return lstm_out, hidden

    def init_hidden(self, batch_size):
        hidden = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        cell = Variable(next(self.parameters()).data.new(
            self.num_layers, batch_size, self.hidden_size))
        return (hidden, cell)

model = BaselineModel(batch_size=32)
optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001)
loss_fn = torch.nn.MSELoss(reduction='sum')

hidden = model.init_hidden(10)
model.zero_grad()
pred, hidden = model(torch.randn(2,10,5), hidden)
pred.size() #torch.Size([2, 10, 5])
outputs = torch.zeros(2,10,5)

loss = loss_fn(pred, outputs)
loss

loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
print(loss)

Please note a common reason for nan values can be related to numerical stability of your learning phase, but usually you have values for the first steps before you see the divergence happening, which is apparently not the case here.


Instruction: How can I implement these bash commands in Google Colab

Input:
I'm a beginner who is working on Neural Machine Translation, the transformer model. I want to implement fairseq Scaling Neural Machine Translation using Google Colab. I guess the commands shown in the README file is written in bash. I know that bash commands can be run in Google Colab by prefixing the command with !. Following commands are from the Github repository mentioned above.

TEXT=wmt16_en_de_bpe32k
mkdir -p $TEXT
tar -xzvf wmt16_en_de.tar.gz -C $TEXT

These commands throw errors when I add the ! as follows.

Output:
Individual bash commands marked by ! are executed in a sub-shell, so variables aren't preserved between lines. If you want to execute a multi-line bash script, use the %%bash cell magic:

%%bash
TEXT=wmt16_en_de_bpe32k
mkdir -p $TEXT
tar -xzvf wmt16_en_de.tar.gz -C $TEXT


Instruction: How to avoid "RuntimeError: error in LoadLibraryA" for torch.cat?

Input:
I am running a  pytorch solution for wireframe detection. I am receiving a "RuntimeError: error in LoadLibraryA" when the solution executes "forward return torch.cat(outputs, 1)"

I am not able to provide a minimal re-producable example. Therefore the quesion: Is it possible to produce just type of error in a microsoft library by python programming errors, or is this most likely a version (of python, pytorch, CUDA,...) problem or a bug in my installation?

I am using windows 10, python 3.8.1 and pytorch 1.4.0.

File "main.py", line 144, in <module>
  main()
File "main.py", line 137, in main
  trainer.train(train_loader, val_loader=None)
File "D:\Dev\Python\Projects\wireframe\wireframe\junc\trainer\balance_junction_trainer.py", line 75, in train
  self.step(epoch, train_loader)
File "D:\Dev\Python\Projects\wireframe\wireframe\junc\trainer\balance_junction_trainer.py", line 176, in step
  ) = self.model(input_var, junc_conf, junc_res, bin_conf, bin_res)
File "D:\Dev\Python\Environment\Environments\pytorch\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
  result = self.forward(*input, **kwargs)
File "D:\Dev\Python\Projects\wireframe\wireframe\junc\model\inception.py", line 41, in forward
  base_feat = self.base_net(im_data)
File "D:\Dev\Python\Environment\Environments\pytorch\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
  result = self.forward(*input, **kwargs)
File "D:\Dev\Python\Projects\wireframe\wireframe\junc\model\networks\inception_v2.py", line 63, in forward
  x = self.Mixed_3b(x)
File "D:\Dev\Python\Environment\Environments\pytorch\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
  result = self.forward(*input, **kwargs)
File "D:\Dev\Python\Projects\wireframe\wireframe\junc\model\networks\inception_v2.py", line 97, in forward
  return torch.cat(outputs, 1)
RuntimeError: error in LoadLibraryA

Output:
Try this workground: run the following code after import torch (should be fixed in 1.5):

import ctypes
ctypes.cdll.LoadLibrary('caffe2_nvrtc.dll')


Instruction: cosine similarity for attention decoder in nmt

Input:
I am implementing a neural machine translation model, and for the decoder part (with attention mechanism) I would like to calculate the cosine similarity for finding the scores.
Here is the function:

score(a,b) =  / ||a|| ||b||

In my case:

        a = htilde_t (N, H)

        b = h (S, N, H)

        the output should be (S, N)

My confusion is about their dimensions and I don't know how to solve that in pytorch.

Output:
See here: https://pytorch.org/docs/master/nn.html?highlight=cosine#torch.nn.CosineSimilarity

cos = nn.CosineSimilarity(dim=2, eps=1e-6)
output = cos(a.unsqueeze(0),b)

you need to unsqueeze to add a ghost dimension to have both input of same dim:

    Input1: (∗1,D,∗2) where D is at position dim

    Input2: (∗1,D,∗2) , same shape as the Input1

    Output: (∗1,∗2)


Instruction: Pytorch: why print(model) does not show the activation functions?

Input:
I need to extract weights, bias and at least the type of activation function from a trained NN in pytorch.

I know that to extract the weights and biases the command is:

model.parameters()

but I can't figure out how to extract also the activation function used on the layers.Here is my network

class NetWithODE(torch.nn.Module):
    def __init__(self, n_feature, n_hidden, n_output, sampling_interval, scaler_features):
        super(NetWithODE, self).__init__()
        self.hidden = torch.nn.Linear(n_feature, n_hidden)  # hidden layer
        self.predict = torch.nn.Linear(n_hidden, n_output)  # output layer
        self.sampling_interval = sampling_interval
        self.device = torch.device("cpu")
        self.dtype = torch.float
        self.scaler_features = scaler_features

    def forward(self, x):
        x0 = x.clone().requires_grad_(True)
        # activation function for hidden layer
        x = F.relu(self.hidden(x))
        # linear output, here r should be the output
        r = self.predict(x)
        # Now the r enters the integrator
        x = self.integrate(r, x0)

        return x

    def integrate(self, r, x0):
        # RK4 steps per interval
        M = 4
        DT = self.sampling_interval / M
        X = x0

        for j in range(M):
            k1 = self.ode(X, r)
            k2 = self.ode(X + DT / 2 * k1, r)
            k3 = self.ode(X + DT / 2 * k2, r)
            k4 = self.ode(X + DT * k3, r)
            X = X + DT / 6 * (k1 + 2 * k2 + 2 * k3 + k4)

        return X

    def ode(self, x0, r):
        qF = r[0, 0]
        qA = r[0, 1]
        qP = r[0, 2]
        mu = r[0, 3]

        FRU = x0[0, 0]
        AMC = x0[0, 1]
        PHB = x0[0, 2]
        TBM = x0[0, 3]

        fFRU = qF * TBM
        fAMC = qA * TBM
        fPHB = qP - mu * PHB
        fTBM = mu * TBM

        return torch.stack((fFRU, fAMC, fPHB, fTBM), 0)

if I run the command

print(model)

I get

NetWithODE(
  (hidden): Linear(in_features=4, out_features=10, bias=True)
  (predict): Linear(in_features=10, out_features=4, bias=True)
)

But where can I get the activation function (in this case Relu)?

I have pytorch 1.4.

Output:
There are two ways of adding operations to the network graph: lowlevel functional way and more advanced object way. You need latter to make your structure observable, In first case is just calling (not exactly, but...) a function without storing info about it. So, instead of

    def forward(self, x):
    ...
        x = F.relu(self.hidden(x))

it must be something like

def __init__(...):
    ...
    self.myFirstRelu= torch.nn.ReLU()

def forward(self, x):
    ...
    x1 = self.hidden(x)
    x2 = self.myFirstRelu(x1)

Anyway a mix of two theese ways is generally bad idea, although even torchvision models have such inconsistiencies: models.inception_v3 not register the poolings for example >:-( (EDIT: it is fixed in june 2020, thanks, mitmul!).

UPD:
- Thanks, that works, now if I print I see ReLU(). But this seems to only print the function in the same order they are defined in the init. Is there a way to get the associations between layers and activation functions? For example I want to know which activation was applyed to layer 1, which to layer 2 end so on...

There is no uniform way, but here is some tricks:
object way:

-just init them in order

-use torch.nn.Sequential

-hook callbacks on nodes like that -

def hook( m, i, o):
    print( m._get_name() )

for ( mo ) in model.modules():
    mo.register_forward_hook(hook)

functional and object way:

-make use of internal model graph, builded on forward pass, as torchviz do (https://github.com/szagoruyko/pytorchviz/blob/master/torchviz/dot.py), or just use plot generated by said torchviz.


Instruction: With a PyTorch LSTM, can I have a different hidden_size than input_size?

Input:
I have:

    def __init__(self, feature_dim=15, hidden_size=5, num_layers=2):
        super(BaselineModel, self).__init__()
        self.num_layers = num_layers
        self.hidden_size = hidden_size

        self.lstm = nn.LSTM(input_size=feature_dim,
                            hidden_size=hidden_size, num_layers=num_layers)

and then I get an error:

RuntimeError: The size of tensor a (5) must match the size of tensor b (15) at non-singleton dimension 2

If I set the two sizes to be the same, then the error goes away. But I'm wondering if my input_size is some large number, say 15, and I want to reduce the number of hidden features to 5, why shouldn't that work?

Output:
It should work the error probably came from elsewhere.
This work for example:

        feature_dim = 15
        hidden_size = 5
        num_layers = 2
        seq_len = 5
        batch_size = 3
        lstm = nn.LSTM(input_size=feature_dim,
                                    hidden_size=hidden_size, num_layers=num_layers)

        t1 = torch.from_numpy(np.random.uniform(0,1,size=(seq_len, batch_size, feature_dim))).float()
        output, states = lstm.forward(t1)
        hidden_state, cell_state = states
        print("output: ",output.size())
        print("hidden_state: ",hidden_state.size())
        print("cell_state: ",cell_state.size())

and return

    output:  torch.Size([5, 3, 5])
    hidden_state:  torch.Size([2, 3, 5])
    cell_state:  torch.Size([2, 3, 5])

Are you using the output somewhere after the lstm ? Did you notice it has a size equal to hidden dim ie 5 on last dim ? It looks like you're using it afterwards thinking it has a size of 15 instead


Instruction: Pytorch: merging two models (nn.Module)

Input:
I have a model that's quite complicated and I therefore can't just call self.fc.weight etc. so I want to iterate over the model in some way.

The goal is to merge models this way: m = alpha * n + (1 - alpha) * o where m n and o are instances of the same class but trained differently. So for each parameter in these models, I want to assign initial values to m based on n and o as described in the equation and then continue the training procedure with m only.

I tried:

for p1, p2, p3 in zip(m.parameters(), n.parameters(), o.parameters()):
    p1 = alpha * p2 + (1 - alpha) * p3

But this does not assign new values within m.

for p1, p2, p3 in zip(m.parameters(), n.parameters(), o.parameters()):
    p1.fill_(alpha * p2 + (1 - alpha) * p3)

But this throws

  RuntimeError: a leaf Variable that requires grad has been used in an in-place operation.

And so I resorted to a working

m.load_state_dict({
    k: alpha * v1 + (1 - alpha) * v2
    for (k, v1), (_, v2) in zip(n.state_dict().items(), o.state_dict().items())
})

Is there a better way to do this in Pytorch? Is it possible that I get gradient errors?

Output:
If I understand you correctly, then you need to get out from under PyTorch's autograd mechanics, which you can by simply doing

p1.data = alpha * p2.data+ (1 - alpha) * p3.data

The parameter's data is not in the parameter itself, but in the data member.


Instruction: Is it possible to add a trainable filter after an autoencoder?

Input:
So I’m building a denoiser with an autoencoder. The idea is that before computing my loss (after the autoencoder), I apply an empirical wiener filter to a texture map of the image and add it back to my autoencoder output (adding back ‘lost detail’). I’ve coded this filter with PyTorch.

My first attempt worked by adding the filter to the end of my autoencoder’s forward function. I can train this network and it backpropagates through my filter in training. However, if I print my network, the filter is not listed, and torchsummary doesn’t include it when calculating parameters.

This has me thinking that I am only training the autoencoder and my filter is filtering the same way every time and not learning.

Is what I’m trying to do possible?

Below is my Autoencoder:

class AutoEncoder(nn.Module):
"""Autoencoder simple implementation """
def __init__(self):
    super(AutoEncoder, self).__init__()
    # Encoder
    # conv layer
    self.block1 = nn.Sequential(
        nn.Conv2d(1, 48, 3, padding=1),
        nn.Conv2d(48, 48, 3, padding=1),
        nn.MaxPool2d(2),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)

    )
    self.block2 = nn.Sequential(
        nn.Conv2d(48, 48, 3, padding=1),
        nn.MaxPool2d(2),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)
    )
    self.block3 = nn.Sequential(
        nn.Conv2d(48, 48, 3, padding=1),
        nn.ConvTranspose2d(48, 48, 2, 2, output_padding=1),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)
    )
    self.block4 = nn.Sequential(
        nn.Conv2d(96, 96, 3, padding=1),
        nn.Conv2d(96, 96, 3, padding=1),
        nn.ConvTranspose2d(96, 96, 2, 2),
        nn.BatchNorm2d(96),
        nn.LeakyReLU(0.1)
    )
    self.block5 = nn.Sequential(
        nn.Conv2d(144, 96, 3, padding=1),
        nn.Conv2d(96, 96, 3, padding=1),
        nn.ConvTranspose2d(96, 96, 2, 2),
        nn.BatchNorm2d(96),
        nn.LeakyReLU(0.1)
    )
    self.block6 = nn.Sequential(
        nn.Conv2d(97, 64, 3, padding=1),
        nn.BatchNorm2d(64),
        nn.Conv2d(64, 32, 3, padding=1),
        nn.BatchNorm2d(32),
        nn.Conv2d(32, 1, 3, padding=1),
        nn.LeakyReLU(0.1)
    )

    # self.blockNorm = nn.Sequential(
    #     nn.BatchNorm2d(1),
    #     nn.LeakyReLU(0.1)
    # )

def forward(self, x):
    # torch.autograd.set_detect_anomaly(True)
    # print("input: ", x.shape)
    pool1 = self.block1(x)
    # print("pool1: ", pool1.shape)
    pool2 = self.block2(pool1)
    # print("pool2: ", pool2.shape)
    pool3 = self.block2(pool2)
    # print("pool3: ", pool3.shape)
    pool4 = self.block2(pool3)
    # print("pool4: ", pool4.shape)
    pool5 = self.block2(pool4)
    # print("pool5: ", pool5.shape)
    upsample5 = self.block3(pool5)
    # print("upsample5: ", upsample5.shape)
    concat5 = torch.cat((upsample5, pool4), 1)
    # print("concat5: ", concat5.shape)
    upsample4 = self.block4(concat5)
    # print("upsample4: ", upsample4.shape)
    concat4 = torch.cat((upsample4, pool3), 1)
    # print("concat4: ", concat4.shape)
    upsample3 = self.block5(concat4)
    # print("upsample3: ", upsample3.shape)
    concat3 = torch.cat((upsample3, pool2), 1)
    # print("concat3: ", concat3.shape)
    upsample2 = self.block5(concat3)
    # print("upsample2: ", upsample2.shape)
    concat2 = torch.cat((upsample2, pool1), 1)
    # print("concat2: ", concat2.shape)
    upsample1 = self.block5(concat2)
    # print("upsample1: ", upsample1.shape)
    concat1 = torch.cat((upsample1, x), 1)
    # print("concat1: ", concat1.shape)
    output = self.block6(concat1)

    t_map = x - output

    for i in range(4):
        tensor = t_map[i, :, :, :]                 # Take each item in batch separately. Could account for this in Wiener instead

        tensor = torch.squeeze(tensor)              # Squeeze for Wiener input format

        tensor = wiener_3d(tensor, 0.05, 10)        # Apply Wiener with specified std and block size
        tensor = torch.unsqueeze(tensor, 0)         # unsqueeze to put back into block
        t_map[i, :, :, :] = tensor                  # put back into block

    filtered_output = output + t_map
    return filtered_output

The for loop at the end is to apply the filter to each image in the batch. I get that this isn’t parallelisable so if anyone has ideas for this, I’d appreciate it. I can post the ‘wiener 3d()’ filter function if that helps, just want to keep the post short.

I’ve tried to define a custom layer class with the filter inside it but I got lost very quickly.

Any help would be greatly appreciated!

Output:
If all you want is to turn your Wiener filter into a module, the following would do:

class WienerFilter(T.nn.Module):
    def __init__(self, param_a=0.05, param_b=10):
        super(WienerFilter, self).__init__()
        # This can be accessed like any other member via self.param_a
        self.register_parameter("param_a", T.nn.Parameter(T.tensor(param_a)))
        self.param_b = param_b

    def forward(self, input):
        for i in range(4):
            tensor = input[i]
            tensor = torch.squeeze(tensor)
            tensor = wiener_3d(tensor, self.param_a, self.param_b)
            tensor = torch.unsqueeze(tensor, 0)
            input[i] = tensor
        return input

You can apply this by adding a line

self.wiener_filter = WienerFilter()

in the init function of your AutoEncoder.

in the forward then you all it by replacing the for loop with

filtered_output = output + self.wiener_filter(t_map)

Torch knows that the wiener_filter module is a member module so it will list the module if you print your AutoEncoder's modules.

If you want to parallelize your wiener filter, you need to do that in PyTorch's terms, meaning using its operations on tensors. Those operations are implemented in a parallel fashion.


Instruction: RuntimeError: Expected object of scalar type Double but got scalar type Float for argument #2

Input:
I have a PyTorch LSTM model and my forward function looks like:

    def forward(self, x, hidden):
        print('in forward', x.dtype, hidden[0].dtype, hidden[1].dtype)
        lstm_out, hidden = self.lstm(x, hidden)
        return lstm_out, hidden

All of the print statements show torch.float64, which I believe is a double. So then why am I getting this issue?

I've cast to double in all of the relevant places already.

Output:
Make sure both your data and model are in dtype double.
For the model:
net = net.double()

For the data:
net(x.double())

It has been discussed on PyTorch forum.


Instruction: using Fasterrcnn for regression or for only finding the bounding box in an image

Input:
Is it possible to use FasterRcnn in pytorch just for finding the bounding box without considering the classification part ?
And is it possible to change the loss of the classification part ( categorical cross entropy ) to a regression loss ( MSE ) ?

Output:
For the first part of your question, please have a look at the line where the bounding boxes are received in demo.

https://github.com/jwyang/faster-rcnn.pytorch/blob/31ae20687b1b3486155809a57eeb376259a5f5d4/demo.py#L297

rois, cls_prob, bbox_pred, \
      rpn_loss_cls, rpn_loss_box, \
      RCNN_loss_cls, RCNN_loss_bbox, \
      rois_label = fasterRCNN(im_data, im_info, gt_boxes, num_boxes)

      scores = cls_prob.data
      boxes = rois.data[:, :, 1:5]

For the second part, yes, it is possible. But you have to make sure that a regression loss such as is suitable for the task that you are planning to use the network for.


Instruction: PyTorch: Simple feedforward neural network not running without retain_graph=True

Input:
Below is my code for training a Feedforward neural network (FFNN).

The labels are numbers between 0 and 50. The FFNN comprises of a single hidden layer with 50 neurons and an output layer with 51 neurons. Furthermore, I have used negative log likelihood loss.

I am very new to PyTorch so I used a couple of websites for guidance. The strange thing is that none of them required retain_graph to be set to True (they dont pass any arguments when calling backward()). Furthermore, it runs very slowly and the accuracy seems to be fluctuating around a fixed value instead of reducing.

Assuming that the input's format is correct, can someone please explain to me why the network is performing so badly and why the network requires retain_graph to be set to True?

Thank you very much!

n_epochs = 2
batch_size = 100
for epoch in range(n_epochs):
    permutation = torch.randperm(training_set.size()[0])
    for i in range(0, training_set.size()[0], batch_size):
        opt.zero_grad()
        indices = permutation[i:i + batch_size]
        batch_features = training_set[indices]
        batch_labels = torch.LongTensor([label for label, sent in train[indices]])
        batch_outputs = model(batch_features)
        loss = loss_function(batch_outputs, batch_labels)
        loss.backward(retain_graph=True)
        opt.step()

Output:
You are missing .zero_grad() operation. Add that to the loop and your code will work fine without retain_graph= True.

loss.backward()
opt.step()
opt.zero_grad()