Instruction: How do I use a .pickle file to predict an image?

Input:
I have trained a CNN model in PyTorch to detect skin diseases in 6 different classes. My model came out with an accuracy of 92% and I saved it in a .pickle file. I wish to use this model for predictions but I don't know how to do so. If anyone can aid me in the necessary steps, I will be grateful. I have tried using Streamlit but apparently, Streamlit does not work anymore so I am opting for an offline solution where I can just upload an image and the model will give me a prediction of such.
Here is the code for my model. I used a pre-trained ResNet18 model and trained it on the Skin Cancer MNIST: HAM10000 dataset from Kaggle.
def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False

def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
    # Initialize these variables which will be set in this if statement. Each of these
    #   variables is model specific.
    model_ft = None
    input_size = 0

    if model_name == "resnet":
        """ Resnet18, resnet34, resnet50, resnet101
        """
        model_ft = models.resnet18(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "vgg":
        """ VGG11_bn
        """
        model_ft = models.vgg11_bn(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier[6].in_features
        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
        input_size = 224

    elif model_name == "densenet":
        """ Densenet121
        """
        model_ft = models.densenet121(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier.in_features
        model_ft.classifier = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "inception":
        """ Inception v3
        """
        model_ft = models.inception_v3(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        # Handle the auxilary net
        num_ftrs = model_ft.AuxLogits.fc.in_features
        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
        # Handle the primary net
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs,num_classes)
        input_size = 299

    else:
        print("Invalid model name, exiting...")
        exit()
    return model_ft, input_size

# resnet,vgg,densenet,inception
model_name = 'resnet'
num_classes = 7
feature_extract = False
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)
# Define the device:
device = torch.device('cuda:0')
# Put the model on the device:
model = model_ft.to(device)

# norm_mean = (0.49139968, 0.48215827, 0.44653124)
# norm_std = (0.24703233, 0.24348505, 0.26158768)
# define the transformation of the train images.
train_transform = transforms.Compose([transforms.Resize((input_size,input_size)),transforms.RandomHorizontalFlip(),
                                      transforms.RandomVerticalFlip(),transforms.RandomRotation(20),
                                      transforms.ColorJitter(brightness=0.1, contrast=0.1, hue=0.1),
                                        transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std)])
# define the transformation of the val images.
val_transform = transforms.Compose([transforms.Resize((input_size,input_size)), transforms.ToTensor(),
                                    transforms.Normalize(norm_mean, norm_std)])

# Define a pytorch dataloader for this dataset
class HAM10000(Dataset):
    def __init__(self, df, transform=None):
        self.df = df
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, index):
        # Load data and get label
        X = Image.open(self.df['path'][index])
        y = torch.tensor(int(self.df['cell_type_idx'][index]))

        if self.transform:
            X = self.transform(X)

        return X, y

# Define the training set using the table train_df and using our defined transitions (train_transform)
training_set = HAM10000(df_train, transform=train_transform)
train_loader = DataLoader(training_set, batch_size=64, shuffle=True, num_workers=4)
# Same for the validation set:
validation_set = HAM10000(df_val, transform=train_transform)
val_loader = DataLoader(validation_set, batch_size=64, shuffle=False, num_workers=4)

# we use Adam optimizer, use cross entropy loss as our loss function
optimizer = optim.Adam(model.parameters(), lr=1e-5)
criterion = nn.CrossEntropyLoss().to(device)

Here are the training process and the save file.
class AverageMeter(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

total_loss_train, total_acc_train = [],[]
def train(train_loader, model, criterion, optimizer, epoch):
    model.train()
    train_loss = AverageMeter()
    train_acc = AverageMeter()
    curr_iter = (epoch - 1) * len(train_loader)
    for i, data in enumerate(train_loader):
        images, labels = data
        N = images.size(0)
        # print('image shape:',images.size(0), 'label shape',labels.size(0))
        images = Variable(images).to(device)
        labels = Variable(labels).to(device)

        optimizer.zero_grad()
        outputs = model(images)

        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        prediction = outputs.max(1, keepdim=True)[1]
        train_acc.update(prediction.eq(labels.view_as(prediction)).sum().item()/N)
        train_loss.update(loss.item())
        curr_iter += 1
        if (i + 1) % 100 == 0:
            print('[epoch %d], [iter %d / %d], [train loss %.5f], [train acc %.5f]' % (
                epoch, i + 1, len(train_loader), train_loss.avg, train_acc.avg))
            total_loss_train.append(train_loss.avg)
            total_acc_train.append(train_acc.avg)
    return train_loss.avg, train_acc.avg

def validate(val_loader, model, criterion, optimizer, epoch):
    model.eval()
    val_loss = AverageMeter()
    val_acc = AverageMeter()
    with torch.no_grad():
        for i, data in enumerate(val_loader):
            images, labels = data
            N = images.size(0)
            images = Variable(images).to(device)
            labels = Variable(labels).to(device)

            outputs = model(images)
            prediction = outputs.max(1, keepdim=True)[1]

            val_acc.update(prediction.eq(labels.view_as(prediction)).sum().item()/N)

            val_loss.update(criterion(outputs, labels).item())

    print('------------------------------------------------------------')
    print('[epoch %d], [val loss %.5f], [val acc %.5f]' % (epoch, val_loss.avg, val_acc.avg))
    print('------------------------------------------------------------')
    return val_loss.avg, val_acc.avg

if os.path.exists("Tested_model2.pickle"):
  print("Loading Trained Model")
  model = pickle.load(open("Tested_model2.pickle", "rb"))
  print(model)
else:
  print("Training New Model.")
  print("Training begins.")
  print("********************************************************")
  epoch_num = 25
  load_model = True
  best_val_acc = 0
  total_loss_val, total_acc_val = [],[]
  for epoch in range(1, epoch_num+1):
    loss_train, acc_train = train(train_loader, model, criterion, optimizer, epoch)
    loss_val, acc_val = validate(val_loader, model, criterion, optimizer, epoch)
    total_loss_val.append(loss_val)
    total_acc_val.append(acc_val)
    if acc_val > best_val_acc:
        best_val_acc = acc_val
        print('*****************************************************')
        print('best record: [epoch %d], [val loss %.5f], [val acc %.5f]' % (epoch, loss_val, acc_val))
        print('*****************************************************')

with open ("Tested_model2.pickle", "wb") as file:
  pickle.dump(model, file)

To put it simply, I wanna know how to use the pickle file for predictions.
Edit: I added the evaluation part in the following, please help me to understand how can I proceed further with this code.
model.eval()
y_label = []
y_predict = []
with torch.no_grad():
    for i, data in enumerate(val_loader):
        images, labels = data
        N = images.size(0)
        images = Variable(images).to(device)
        outputs = model(images)
        prediction = outputs.max(1, keepdim=True)[1]
        y_label.extend(labels.cpu().numpy())
        y_predict.extend(np.squeeze(prediction.cpu().numpy().T))

Also, This is the code that I previously used to load and make predictions, however I did not get to know whether the code is correct or not or the method is right.
%%writefile app.py
import streamlit as st
import torch

st.set_option('deprecation.showfileUploaderEncoding', False)
@st.cache(allow_output_mutation=True)

def load_model():
  model = pickle.load(open("Trained_Model_part2.pickle", "rb"))
  return mdoel

model = load_model()
st.write("""
         #Classification of skin disease
         """)

file = st.file_uploader("Please upload the image of the affected area.", type = ["jpg", "png"])

import cv2
from PIL import Image, ImageOps
import numpy as np

def import_and_predict(image_data, model):
  size = (224, 224)
  image = ImageOps.fit(image_data, size, Image.ANTIALIAS)
  img = np.asarray(image)
  image_reshape = img[np.newaxis,...]
  prediction = model.predict(img_reshape)

  return prediction

if file is None:
  st.text("Please upload an image file.")
else:
  image = Image.open(file)
  st.image(image, use_column_width = True)
  predictions = import_and_predict(image, model)
  class_names = ["Melanocytic nevi", "Melanoma", "Benign keratosis-like lesions", "Basal cell carcinoma", "Actinic keratoses", "Vascular lesions", "Dermatofibroma"]
  string = "It is: " + class_names[np.argmax(predictions)]
  st.success(string)

This uses streamlit and the loader is the previous pickle file loader which will be replaced by a .pth loader. I want to know what changes do I have to make so the code will ask for an image input or look for an image in the specific folder and deliver a prediction. Thank you.

Output:
I'll show you how to save and load pytorch model parameters properly (you should use the .pt extension):
To save the model do this (once every epoch or after training):
torch.save(model.state_dict(), "your/path/model_file.pt")

All the model parameters are now loaded into "your/path/model_file.pt".
To load the model now you will need the model class (class YourModel(nn.Module): ...) and the parameters:
model = YourModel()
model.load_state_dict(torch.load("your/path/model_file.pt"))

The model is now initialized with the trianed parameters and ready to use. For example like this:
model = YourModel()
model.load_state_dict(torch.load("your/path/model_file.pt"))

# set to evaluation mode
model.eval()

# load an image
sample = get_sample()

# reshape sample to (batch-size x width x height) but batch-size is 1 because you probably want to predict just one image at a time in real-life usage
sample = torch.reshape(1, sample.size(0), sample.size(1))

prediction = model(sample)

Edit to answer the question in the comments:
To load a trained pytorch model you need the file in which the models parameter is saved and the model structure itself. The model structure is just the python code of a pytorch module class. You havent build the model youself therefore you dont have the direct model code but in your case it should be model_ft. It's just the python class holding all the layers. So the model class is like the skeletton and the parameters like the flesh or something.
When you would create a model class completly yourself and load the trained weights into it it would look like this for example:

import torch
import torch.nn as nn
import torch.nn.functional as F

# the model (skeleton class)
class YourModel(nn.Module):
    def __init__(self):
        super(YourModel, self).__init__()

        self.dense1 = nn.Linear(128, 64)
        self.dense2 = nn.Linear(64, 2)

    def forward(self, x):
        x = F.relu(self.dense1(x))
        x = torch.sigmoid(self.dense2(x))

        return x

# . . .
# train model and save it to model.pt
# . . .

# load "empty" model
model = YourModel()

# load trained paramters/weights into the model
model.load_state_dict(torch.load("/path/model.pt"))

So as I said, in your case it the model class should be model_ft.


Instruction: Pytorch : RuntimeError: mat1 dim 1 must match mat2 dim 0

Input:
using resnet50 model. Customize the last layer and it showing runtime error..Im new to PyTorch and I keep getting the error mat1 dim1 must match mat1 dim0
this is my code for the network
from torchvision import models
model = models.resnet50(pretrained=True)

for param in model.parameters():
    param.requires_grad = False

class Identity(nn.Module):
    def __init__(self):
        super(Identity, self).__init__()
    def forward(self, x):
        return x

model.avgpool = Identity()
model.fc = nn.Linear(2048, 2, bias=True)

for param in model.fc.parameters():
    param.requires_grad = True
model = model.to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)

def train(num_epoch, model):
    for epoch in range(0, 3):
        losses = []
        model.train()
        loop = tqdm(enumerate(train_loader), total=len(train_loader))
        for batch_idx, (data, targets) in loop:
            data = data.to(device=device)
            targets = targets.to(device=device)
            scores = model.forward(data)
            loss = criterion(scores, targets)

            optimizer.zero_grad()
            losses.append(loss)
            loss.backward()
            optimizer.step()

            loop.set_description(f"Epoch {epoch+1}/{num_epoch} process: {int((batch_idx / len(train_loader)) * 100)}")
            loop.set_postfix(loss=loss.data.item())

train(1, model)

RuntimeError: mat1 dim 1 must match mat2 dim 0

Output:
This error comes from the nn.Linear you changed.
As you recall, nn.Linear computes a simple matrix dot product, and therefore the input dimension coming from the previous layer must equal the weight matrix shape (you set it to 2048).
my guess is that since you removed the model.avgpool layer, you now have more than 2048 input dimension resulting with the error you got.

BTW, you do not need to implement "identity" layer yourself, pytorch already has nn.Identity.


Instruction: create tensor in specific indexes without loops

Input:
I have a tensor t1 (with shape (2*n, 2*n), and I need to create tensor t2 (with shape (2*n)) with the values of t1 at [i,(i+n) mod 2n] for each row i.
For example, given:
t1 = torch.tensor([[1, 2, 3, 4],
                   [5, 6, 7, 8],
                   [9 ,10,11,12],
                   [13,14,15,16]])

Here n=2.
t2 has to be [3,8,9,14].
I have this code:
t2 = torch.tensor([t1[i,(i+n)%(2*n)] for i in range(2*n)])

but Im searching a way without loops.

Output:
You could create the list of indices beforehand with a loop then pick the values from t1 with torch.gather without having to loop over them yourself:
>>> index = torch.tensor([[(i+n)%(2*n)] for i in range(2*n)])

>>> torch.gather(t1, 1, index).flatten()
tensor([ 3,  8,  9, 14])

Alternatively, you can use torch.roll to shift by n an (2*n, 2*n) identity matrix, multiply by t1 to get a mask and apply that to t1 with torch.masked_select:
>>> mask = torch.eye(2*n).roll(n, 1).bool()
tensor([[False, False,  True, False],
        [False, False, False,  True],
        [ True, False, False, False],
        [False,  True, False, False]])

>>> torch.masked_select(t1, mask)
tensor([ 3,  8,  9, 14])


Instruction: what does offsets mean in pytorch nn.EmbeddingBag?

Input:
I know offsets meaning when it has two numbers, but what does it mean when more than two numbers,for example:
weight = torch.FloatTensor([[1, 2, 3], [4, 5, 6]])
embedding_sum = nn.EmbeddingBag.from_pretrained(weight, mode='sum')
print(list(embedding_sum.parameters()))
input = torch.LongTensor([0,1])
offsets = torch.LongTensor([0,1,2,1])

print(embedding_sum(input, offsets))

the result is :
[Parameter containing:
tensor([[1., 2., 3.],
        [4., 5., 6.]])]
tensor([[1., 2., 3.],
        [4., 5., 6.],
        [0., 0., 0.],
        [0., 0., 0.]])

who can help me?

Output:
import torch
import torch.nn as nn

weight = torch.FloatTensor([[1, 2, 3], [4, 5, 6]])
embedding_sum = nn.EmbeddingBag.from_pretrained(weight, mode='sum')
print(embedding_sum.weight)

""" output
Parameter containing:
tensor([[1., 2., 3.],
        [4., 5., 6.]])
"""

input = torch.LongTensor([0, 1])
offsets = torch.LongTensor([0, 1, 2, 1])

According to these offsets you will get the following samples
"""
sample_1: input[0:1] # tensor([0])
sample_2: input[1:2] # tensor([1])
sample_3: input[2:1] # tensor([])
sample_4: input[1:]  # tensor([1])
"""

Embedding the samples above
# tensor([0]) => lookup 0  => embedding_sum.weight[0] => [1., 2., 3.]
# tensor([1]) => lookup 1  => embedding_sum.weight[1] => [4., 5., 6.]
# tensor([])  => empty bag                            => [0., 0., 0.]
# tensor([1]) => lookup 1  => embedding_sum.weight[1] => [4., 5., 6.]

print(embedding_sum(input, offsets))

""" output
tensor([[1., 2., 3.],
        [4., 5., 6.],
        [0., 0., 0.],
        [4., 5., 6.]])
"""

One more example:
input = torch.LongTensor([0, 1])
offsets = torch.LongTensor([0, 1, 0])

According to these offsets you will get the following samples
"""
sample_1: input[0:1] # tensor([0])
sample_2: input[1:0] # tensor([])
sample_3: input[0:]  # tensor([0, 1])
"""

Embedding the samples above
# tensor([0])    => lookup 0 => embedding_sum.weight[0] => [1., 2., 3.]
# tensor([])     => empty bag => [0., 0., 0.]
# tensor([0, 1]) => lookup 0 and 1 then reduce by sum
#                => embedding_sum.weight[0] + embedding_sum.weight[1] => [5., 7., 9.]

print(embedding_sum(input, offsets))

""" output
tensor([[1., 2., 3.],
        [0., 0., 0.],
        [5., 7., 9.]])
"""


Instruction: How do I to average irregularly spaced x & y coordinate tensor into a grid with a specific cell size?

Input:
I have an algorithm that generates a tensor of irregularly spaced x and y coordinates (ex: torch.size([3600, 2])), and I need to average the points into grid cells of a specific size (ex: 8 by 8). The resulting grid needs to be either an array or tensor.
It's not required, but I would also like to be able to determine if any of the resulting cells have less than a specified number of points in them.
For example I can graph the tensor using matplotlib's plt.scatter, and it looks like this:

In the above example, 100,000 points exist but the number of points can sometimes be in the tens of millions.
I've tried using histogram approaches, and most of them use a specific number of cells vs a specific cell size. Matplotlib can seemingly do it in a graph, but that doesn't help me get an array or tensor.
Edit:
This code might work, if it can be made to work properly.
def grid_torch(x_coords, y_coords, grid_size=(8,8), x_extent=(0., 1.), y_extent=(0., 1.)):
    x_coords = ((x_coords - x_extent[0]) / (x_extent[1] - x_extent[0])) * grid_size[0]
    y_coords = ((y_coords - y_extent[0]) / (y_extent[1] - y_extent[0])) * grid_size[1]

    x_list = []
    for x in range(grid_size[0]):
        x = torch.ones_like(x_coords) * x
        y_list = []
        for y in range(grid_size[1]):
            y = torch.ones_like(y_coords) * y
            in_bounds_x = torch.logical_and(x <= x_coords, x_coords <= x + 1)
            in_bounds_y = torch.logical_and(y <= y_coords, y_coords <= y + 1)
            in_bounds = torch.logical_and(in_bounds_x, in_bounds_y)

            in_bounds_indices = torch.where(in_bounds)
            print(in_bounds_indices)
            y_list.append(in_bounds_indices)
        x_list.append(torch.stack(y_list))
    return torch.stack(x_list)

out = grid_torch(xy_tensor[:,0], xy_tensor[:,1])

print(out.shape)

def create_grid(grid_layout, activ, grid_size=(8,8), min_density=8):
    cells = []
    for x in range(grid_size[0]):
        for y in range(grid_size[1]):
            indices = grid_layout[x, y]
            if len(indices) > min_density:
                average_activation = torch.mean(activ[indices])
                cells.append((average_activation, x, y))
                print(average_activation, x, y)
    return torch.stack(cells)

grid_test = create_grid(out, xy_tensor, grid_size=(8,8))

Output:
I think this code would give you a good starting point.
def grid_torch(x_coords, y_coords, grid_size=(8,8), x_extent=(0., 1.), y_extent=(0., 1.)):
    # This part converts coordinates to bin numbers (like (2,5), (7,7) etc)
    x_bin = (((x_coords - x_extent[0]) / (x_extent[1] - x_extent[0])) * grid_size[0]).int()
    y_bin = (((y_coords - y_extent[0]) / (y_extent[1] - y_extent[0])) * grid_size[1]).int()

    counts = torch.zeros(grid_size)
    means = torch.zeros(list(grid_size) + [2])
    for x in range(grid_size[0]):
        for y in range(grid_size[1]):
            # these tensors are 1 where (x_bin == x and y_bin == y), 0 else where
            x_where = 1 * (x_bin == x)
            y_where = 1 * (y_bin == y)
            p_where = (x_where * y_where)
            cnt = p_where.sum()
            counts[x, y] = cnt

            # we'll average both x and y coords seperately.
            # you can embed min_density logic here.
            if cnt > 0:
                means[x, y, 0] = (x_coords * p_where).sum() / p_where.sum()
                means[x, y, 1] = (y_coords * p_where).sum() / p_where.sum()

    return counts, means

# Generate sample points
points = torch.tensor(np.concatenate([
    np.random.normal(loc=0.2, scale=0.1, size=(1000, 2)),
    np.random.normal(loc=0.6, scale=0.1, size=(1000, 2))
]).clip(0,1)).float()
# plt.scatter(points[:,0], points[:,1])
# plt.grid()

counts, means = grid_torch(points[:,0], points[:,1])
counts

>>>
tensor([[ 47., 114.,  75.,  10.,   0.,   0.,   0.,   0.],
        [102., 204., 141.,  27.,   0.,   0.,   0.,   0.],
        [ 60., 101.,  74.,  16.,   7.,   4.,   1.,   0.],
        [  5.,  17.,   9.,  23.,  72.,  51.,  10.,   0.],
        [  1.,   1.,   4.,  54., 186., 141.,  28.,   3.],
        [  0.,   0.,   3.,  47., 154., 117.,  14.,   0.],
        [  0.,   0.,   0.,   9.,  37.,  24.,   4.,   0.],
        [  0.,   0.,   0.,   2.,   0.,   1.,   0.,   0.]])


Instruction: PyTorch mini batch, when to call optimizer.zero_grad()

Input:
When we use mini batch, should I call optimizer.zero_grad() before starting the iteration? Or inside the iteration? I think the second code is correct, but I'm not sure.
nb_epochs = 20
    for epoch in range(nb_epochs + 1):
      optimizer.zero_grad() # THIS PART!!
      for batch_idx, samples in enumerate(dataloader):

        x_train, y_train = samples

        prediction = model(x_train)

        cost = F.mse_loss(prediction, y_train)

        cost.backward()
        optimizer.step()

        print('Epoch {:4d}/{} Batch {}/{} Cost: {:.6f}'.format(
            epoch, nb_epochs, batch_idx+1, len(dataloader),
            cost.item()
            ))

or
nb_epochs = 20
    for epoch in range(nb_epochs + 1):

      for batch_idx, samples in enumerate(dataloader):

        x_train, y_train = samples

        prediction = model(x_train)

        optimizer.zero_grad() #THIS PART!!
        cost = F.mse_loss(prediction, y_train)

        cost.backward()
        optimizer.step()

        print('Epoch {:4d}/{} Batch {}/{} Cost: {:.6f}'.format(
            epoch, nb_epochs, batch_idx+1, len(dataloader),
            cost.item()
            ))

Which one is correct? The only difference is location of optimizer.zero_grad().

Output:
Gradients accumulates by default everytime you call .backward() on the computational graph.
On the first snippet, you are resetting the gradients once per epoch so all gradients will accumulate their values over time. With a total of len(dataloader) accumulated gradients, only resseting the gradients when the next epoch starts. On the second snippet, you are doing the right thing, which is to reset the gradient after every backward pass.
So your assumptions were right.
There are some instances where accumulating gradients is needed, but most times it's not.


Instruction: Torch throws a RuntimeError: element 0 of tensors does not require grad... but can't find where computational graph is severed

Input:
I am getting the above error:
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
I looked this up and it looks like the computational graph is not connected for some reason.  However, I cannot find the location where the graph is severed.
My code is a reproduction of the arjovsky WGAN: https://github.com/martinarjovsky/WassersteinGAN
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import torch
import torch.nn as nn

from __future__ import print_function
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import os
import json

class MLP_G(nn.Module):
    def __init__(self, isize, nz, ngf, ngpu):
        super(MLP_G, self).__init__()
        self.ngpu = ngpu

        main = nn.Sequential(
            # Z goes into a linear of size: ngf
            nn.Linear(nz, ngf),
            nn.ReLU(True),
            nn.Linear(ngf, ngf),
            nn.ReLU(True),
            nn.Linear(ngf, ngf),
            nn.ReLU(True),
            nn.Linear(ngf, isize),
        )
        self.main = main
        self.isize = isize
        self.nz = nz

    def forward(self, input):
        input = input.view(input.size(0), input.size(1))
        if isinstance(input.data, torch.cuda.FloatTensor) and self.ngpu > 1:
            output = nn.parallel.data_parallel(self.main, input, range(self.ngpu))
        else:
            output = self.main(input)
        return output.view(output.size(0), self.isize)
class MLP_D(nn.Module):
    def __init__(self, isize, nz, ndf, ngpu):
        super(MLP_D, self).__init__()
        self.ngpu = ngpu

        main = nn.Sequential(
            # Z goes into a linear of size: ndf
            nn.Linear(isize, ndf),
            nn.ReLU(True),
            nn.Linear(ndf, ndf),
            nn.ReLU(True),
            nn.Linear(ndf, ndf),
            nn.ReLU(True),
            nn.Linear(ndf, 1),
        )
        self.main = main
        self.isize = isize
        self.nz = nz

    def forward(self, input):
        input = input.view(input.size(0),input.size(1))
        if isinstance(input.data, torch.cuda.FloatTensor) and self.ngpu > 1:
            output = nn.parallel.data_parallel(self.main, input, range(self.ngpu))
        else:
            output = self.main(input)
        output = output.mean(0)
        return output.view(1)

netG = None #path to saved generator
netD = None #discriminator path
batchSize = 1000 #size of batch (which is size of data)
cuda = False
lrD = lrG = .00005
beta1 = .5
niter = 25
experiment = '/content/drive/MyDrive/savefolder'
clamp_upper = .01
clamp_lower = -clamp_upper

manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", manualSeed)
random.seed(manualSeed)
torch.manual_seed(manualSeed)

cudnn.benchmark = True

dataset = torch.tensor(np.stack([x,y, instrument], axis = 1)).float().reshape(-1,3)

ngpu = 1
nz = 4 #three latents and the instrument
ngf = 128
ndf = 128

# custom weights initialization called on netG and netD
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

netG = MLP_G(2, nz, ngf, ngpu)

netG.apply(weights_init)

print(netG)

netD = MLP_D(3, nz, ndf, ngpu)

print(netD)

input = torch.FloatTensor(batchSize, 2)
noise = torch.FloatTensor(batchSize, nz-1)
fixed_noise = torch.FloatTensor(batchSize, nz-1).normal_(0, 1)
one = torch.FloatTensor([1])
mone = one * -1

# setup optimizer

optimizerD = optim.Adam(netD.parameters(), lr=lrD, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=lrG, betas=(beta1, 0.999))

real_cpu = data = dataset

gen_iterations = 0
for epoch in range(niter):
    #data_iter = iter(dataloader)
    ############################
    # (1) Update D network
    ###########################
    for p in netD.parameters(): # reset requires_grad
        p.requires_grad = True # they are set to False below in netG update

    # train the discriminator Diters times
    if gen_iterations < 25 or gen_iterations % 500 == 0:
        Diters = 100
    else:
        Diters = 5
    j = 0
    while j < Diters:
        j += 1

        # clamp parameters to a cube
        for p in netD.parameters():
            p.data.clamp_(clamp_lower, clamp_upper)

        # train with real
        netD.zero_grad()

        if cuda:
            real_cpu = real_cpu.cuda()
        input.resize_as_(real_cpu).copy_(real_cpu)
        inputv = Variable(input, requires_grad=False)

        errD_real = netD(inputv)
        errD_real.backward(one)#Error Occurs here

        # train with fake
        noise.resize_(batchSize, nz-1).normal_(0, 1)
        noisev = torch.cat([Variable(noise, requires_grad=False), dataset[:,2].reshape(-1,1)], 1)# totally freeze netG
        fake = torch.cat([Variable(netG(noisev).data), dataset[:,2].view(-1,1)], 1)
        inputv = fake
        errD_fake = netD(inputv)
        errD_fake.backward(mone)
        errD = errD_real - errD_fake
        optimizerD.step()

        ############################
        # (2) Update G network
        ###########################
        for p in netD.parameters():
            p.requires_grad = False # to avoid computation
        netG.zero_grad()
        # in case our last batch was the tail batch of the dataloader,
        # make sure we feed a full batch of noise
        noise.resize_(batchSize, nz-1).normal_(0, 1)
        noisev = torch.cat([Variable(noise), dataset[:,2].view(-1,1)], 1)
        fake = torch.cat([netG(noisev), dataset[:,2].view(-1,1)], 1)
        errG = netD(fake)
        errG.backward(one)
        optimizerG.step()
        gen_iterations += 1

        i = 0

        print('[%d/%d][%d] Loss_D: %f Loss_G: %f Loss_D_real: %f Loss_D_fake %f'
            % (epoch, niter, gen_iterations,
            errD.data[0], errG.data[0], errD_real.data[0], errD_fake.data[0]))
        # if gen_iterations % 500 == 0:
        #     real_cpu = real_cpu.mul(0.5).add(0.5)
        #     vutils.save_image(real_cpu, '{0}/real_samples.png'.format(opt.experiment))
        #     fake = netG(Variable(fixed_noise, volatile=True))
        #     fake.data = fake.data.mul(0.5).add(0.5)
        #     vutils.save_image(fake.data, '{0}/fake_samples_{1}.png'.format(opt.experiment, gen_iterations))

    # do checkpointing
    torch.save(netG.state_dict(), '{0}/netG_epoch_{1}.pth'.format(experiment, epoch))
    torch.save(netD.state_dict(), '{0}/netD_epoch_{1}.pth'.format(experiment, epoch))

Error occurs on the line: errD_real.backward(one).  The error might be something regarding zeroing out the computational graph as the code runs for one iteration then throws an error.  Thanks for your help.

Output:
You most certainly need to add require_grad=True on one. You could define it as:
one = torch.tensor([1], dtype=torch.float16, requires_grad=True)


Instruction: Tensorboard: All experiments were written as one (without provided tags)

Input:
I wanted to compare several runs that I did in the loop creating new SummaryWriter instances like this:
for experiment_name in experiments:
    logger = SummaryWriter(self._log_path, comment=experiment_name)
    ...
    for epoch in range(5):
        ...
        logger.add_scalar("Epoch Loss", loss, epoch)
        ...
    logger.close()

In the log path I got several files like this:
events.out.tfevents.1609675249.nlp-vm.13735.0
events.out.tfevents.1609679736.nlp-vm.13735.1
events.out.tfevents.1609687200.nlp-vm.13735.2
events.out.tfevents.1609691662.nlp-vm.13735.3
events.out.tfevents.1609699158.nlp-vm.13735.4
events.out.tfevents.1609703743.nlp-vm.13735.5
events.out.tfevents.1609711308.nlp-vm.13735.6
events.out.tfevents.1609716054.nlp-vm.13735.7

But the Tensorboard displays all runs as one:

example

example

Could you say what I should do to fix it and can I do it without re-running all experiments?

Output:
The events files that I have in the one folder should be in the separate and the folder name will be displayed as an experiment name.
Also found the important note in the SummaryWriter documentation:

comment (string): Comment log_dir suffix appended to the default
log_dir. If log_dir is assigned, this argument has no effect.


Instruction: torchtext ImportError in colab

Input:
I am trying to run this tutorial in colab.
However, when I try to import a bunch of modules:
import io
import torch
from torchtext.utils import download_from_url
from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator

It gives me the errors for extract_archive and build_vocab_from_iterator:
---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-5-a24e72502dbc> in <module>()
      1 import io
      2 import torch
----> 3 from torchtext.utils import download_from_url, extract_archive
      4 from torchtext.data.utils import get_tokenizer
      5 from torchtext.vocab import build_vocab_from_iterator

ImportError: cannot import name 'extract_archive'

ImportError                               Traceback (most recent call last)
<ipython-input-4-02a401fd241b> in <module>()
      3 from torchtext.utils import download_from_url
      4 from torchtext.data.utils import get_tokenizer
----> 5 from torchtext.vocab import build_vocab_from_iterator
      6
      7 url = 'https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip'

ImportError: cannot import name 'build_vocab_from_iterator'

Please help me with this one.

Output:
You need to upgrade torchtext first
!pip install -U torchtext==0.8.0

Currently, version 0.8.0 works with torch 1.7.0 (no need to upgrade torch, torchvision)
Update (sep 2021)
Currently, torchtext is already 0.10.0 and you don't need to upgrade anything.


Instruction: Pytorch transformer forward function masks implementation for decoder forward function

Input:
I am trying to use and learn PyTorch Transformer with DeepMind math dataset. I have tokenized (char not word) sequence that is fed into model. Models forward function is doing once forward for encoder and multiple forwards for decoder (till all batch outputs reach  token, this is still TODO).
I am struggling with Transformer masks and decoder forward as it throws the error:
    k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1)
    RuntimeError: shape '[-1, 24, 64]' is invalid for input of size 819200.

Source is N = 32, S = 50, E = 512. Target is N = 32, S = 3, E = 512.
It is possible that I have wrong implementation of masks or that source and target lengths are different, not realy sure.
class PositionalEncoding(nn.Module):
# function to positionally encode src and target sequencies
def __init__(self, d_model, dropout=0.1, max_len=5000):
    super(PositionalEncoding, self).__init__()
    self.dropout = nn.Dropout(p=dropout)
    pe = torch.zeros(max_len, d_model)
    position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
    div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
    pe[:, 0::2] = torch.sin(position * div_term)
    pe[:, 1::2] = torch.cos(position * div_term)
    pe = pe.unsqueeze(0).transpose(0, 1)
    self.register_buffer('pe', pe)

def forward(self, x):
    x = x + self.pe[:x.size(0), :]
    return self.dropout(x)

class MyTransformerModel(nn.Module):
# should implement init and forward function
# define separate functions for masks
# define forward function with
# implement:
#  embedding layer
#  positional encoding
#  encoder layer
#  decoder layer
#  final classification layer
# encoder -> forward once
# decoder -> forward multiple times (for one encoder forward)
# decoder output => concatenate to input e.g. decoder_input = torch.cat([decoder_input], [decoder_output])
# early stopping => all in batch reach <eos> token
def __init__(self, vocab_length = 30, sequence_length = 512, num_encoder_layers = 3, num_decoder_layers = 2, num_hidden_dimension = 256, feed_forward_dimensions = 1024, attention_heads = 8, dropout = 0.1, pad_idx = 3, device = "CPU", batch_size = 32):
    super(MyTransformerModel, self).__init__()
    self.src_embedding = nn.Embedding(vocab_length, sequence_length)
    self.pos_encoder = PositionalEncoding(sequence_length, dropout)
    self.src_mask = None # attention mask
    self.memory_mask = None # attention mask
    self.pad_idx = pad_idx
    self.device = device
    self.batch_size = batch_size
    self.transformer = nn.Transformer(
        sequence_length,
        attention_heads,
        num_encoder_layers,
        num_decoder_layers,
        feed_forward_dimensions,
        dropout,
    )

def src_att_mask(self, src_len):
    mask = (torch.triu(torch.ones(src_len, src_len)) == 1).transpose(0, 1)
    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
    return mask

def no_peak_att_mask(self, batch_size, src_len, time_step):
    mask = np.zeros((batch_size, src_len), dtype=bool)
    mask[:, time_step: ] = 1 # np.NINF
    mask = torch.from_numpy(mask)
    return mask

def make_src_key_padding_mask(self, src):
    # mask "<pad>"
    src_mask = src.transpose(0, 1) == self.pad_idx
    return src_mask.to(self.device)

def make_trg_key_padding_mask(self, trg):
    tgt_mask = trg.transpose(0, 1) == self.pad_idx
    return tgt_mask.to(self.device)

def forward(self, src, trg):
    src_seq_length, N = src.shape
    trg_seq_length, N = trg.shape
    embed_src = self.src_embedding(src)
    position_embed_src =  self.pos_encoder(embed_src)
    embed_trg = self.src_embedding(trg)
    position_embed_trg = self.pos_encoder(embed_trg)
    src_padding_mask = self.make_src_key_padding_mask(src)
    trg_padding_mask = self.make_trg_key_padding_mask(trg)
    trg_mask = self.transformer.generate_square_subsequent_mask(trg_seq_length).to(self.device)
    time_step = 1
    att_mask = self.no_peak_att_mask(self.batch_size, src_seq_length, time_step).to(self.device)
    encoder_output = self.transformer.encoder.forward(position_embed_src, src_key_padding_mask = src_padding_mask)
    # TODO : implement loop for transformer decoder forward fn, implement early stopping
    # where to feed decoder_output?
    decoder_output = self.transformer.decoder.forward(position_embed_trg, encoder_output, trg_mask, att_mask, trg_padding_mask, src_padding_mask)
    return decoder_output

Can anyone pin point where I have made a mistake?

Output:
It looks like I have messed dimensions order (as Transformer does not have batch first option). Corrected code is below:
class MyTransformerModel(nn.Module):
def __init__(self, d_model = 512, vocab_length = 30, sequence_length = 512, num_encoder_layers = 3, num_decoder_layers = 2, num_hidden_dimension = 256, feed_forward_dimensions = 1024, attention_heads = 8, dropout = 0.1, pad_idx = 3, device = "CPU", batch_size = 32):
    #, ninp, device, nhead=8, nhid=2048, nlayers=2, dropout=0.1, src_pad_idx = 1, max_len=5000, forward_expansion= 4):
    super(MyTransformerModel, self).__init__()
    self.src_embedding = nn.Embedding(vocab_length, d_model)
    self.pos_encoder = PositionalEncoding(d_model, dropout)
    self.vocab_length = vocab_length
    self.d_model = d_model
    self.src_mask = None # attention mask
    self.memory_mask = None # attention mask
    self.pad_idx = pad_idx
    self.device = device
    self.batch_size = batch_size
    self.transformer = nn.Transformer(
        d_model,
        attention_heads,
        num_encoder_layers,
        num_decoder_layers,
        feed_forward_dimensions,
        dropout,
    )

    self.fc = nn.Linear(d_model, vocab_length)
    # self.init_weights() <= used in tutorial

def src_att_mask(self, src_len):
    mask = (torch.triu(torch.ones(src_len, src_len)) == 1).transpose(0, 1)
    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
    return mask

def no_peak_att_mask(self, batch_size, src_len, time_step):
    mask = np.zeros((batch_size, src_len), dtype=bool)
    mask[:, time_step: ] = 1 # np.NINF
    mask = torch.from_numpy(mask)
    # mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
    return mask

def make_src_key_padding_mask(self, src):
    # mask "<pad>"
    src_mask = src.transpose(0, 1) == self.pad_idx
    # src_mask = src == self.pad_idx
    # (N, src_len)
    return src_mask.to(self.device)

def make_trg_key_padding_mask(self, trg):
    # same as above -> expected tgt_key_padding_mask: (N, T)
    tgt_mask = trg.transpose(0, 1) == self.pad_idx
    # tgt_mask = trg == self.pad_idx
    # (N, src_len)
    return tgt_mask.to(self.device)

def init_weights(self):
    initrange = 0.1
    nn.init.uniform_(self.encoder.weight, -initrange, initrange)
    nn.init.zeros_(self.decoder.weight)
    nn.init.uniform_(self.decoder.weight, -initrange, initrange)

def forward(self, src, trg):
    N, src_seq_length = src.shape
    N, trg_seq_length = trg.shape
    #  S - source sequence length
    #  T - target sequence length
    #  N - batch size
    #  E - feature number
    #  src: (S, N, E) (sourceLen, batch, features)
    #  tgt: (T, N, E)
    #  src_mask: (S, S)
    #  tgt_mask: (T, T)
    #  memory_mask: (T, S)
    #  src_key_padding_mask: (N, S)
    #  tgt_key_padding_mask: (N, T)
    #  memory_key_padding_mask: (N, S)
    src = rearrange(src, 'n s -> s n')
    trg = rearrange(trg, 'n t -> t n')
    print("src shape {}".format(src.shape))
    print(src)
    print("trg shape {}".format(trg.shape))
    print(trg)

    embed_src = self.src_embedding(src)
    print("embed_src shape {}".format(embed_src.shape))
    print(embed_src)
    position_embed_src =  self.pos_encoder(embed_src)
    print("position_embed_src shape {}".format(position_embed_src.shape))
    print(position_embed_src)
    embed_trg = self.src_embedding(trg)
    print("embed_trg shape {}".format(embed_trg.shape))
    print(embed_trg)
    position_embed_trg = self.pos_encoder(embed_trg)
    # position_embed_trg = position_embed_trg.transpose(0, 1)
    print("position_embed_trg shape {}".format(position_embed_trg.shape))
    print(position_embed_trg)
    src_padding_mask = self.make_src_key_padding_mask(src)
    print("KEY - src_padding_mask shape {}".format(src_padding_mask.shape))
    print("should be of shape: src_key_padding_mask: (N, S)")
    print(src_padding_mask)
    trg_padding_mask = self.make_trg_key_padding_mask(trg)
    print("KEY - trg_padding_mask shape {}".format(trg_padding_mask.shape))
    print("should be of shape: trg_key_padding_mask: (N, T)")
    print(trg_padding_mask)
    trg_mask = self.transformer.generate_square_subsequent_mask(trg_seq_length).to(self.device)
    print("trg_mask shape {}".format(trg_mask.shape))
    print("trg_mask should be of shape tgt_mask: (T, T)")
    print(trg_mask)
    # att_mask = self.src_att_mask(trg_seq_length).to(self.device)
    time_step = 1
    # error => memory_mask: expected shape! (T, S) !!! this is not a key_padding_mask!
    # att_mask = self.no_peak_att_mask(self.batch_size, src_seq_length, time_step).to(self.device)
    # print("att_mask shape {}".format(att_mask.shape))
    # print("att_mask should be of shape  memory_mask: (T, S)")
    # print(att_mask)
    att_mask = None
    # get encoder output
    # forward(self, src: Tensor, mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None)
    # forward encoder just once for a batch
    # attention forward of encoder expects => src, src_mask, src_key_padding_mask +++ possible positional encoding error !!!
    encoder_output = self.transformer.encoder.forward(position_embed_src, src_key_padding_mask = src_padding_mask)
    print("encoder_output")
    print("encoder_output shape {}".format(encoder_output.shape))
    print(encoder_output)
    # forward decoder till all in batch did not reach <eos>?
    # def forward(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None,
    # memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None,
    # memory_key_padding_mask: Optional[Tensor] = None)
    # first forward
    decoder_output = self.transformer.decoder.forward(position_embed_trg, encoder_output, trg_mask, att_mask, trg_padding_mask, src_padding_mask)
    # TODO: target in => target out shifted by one, loop till all in batch meet stopping criteria || max len is reached
    #
    print("decoder_output")
    print("decoder_output shape {}".format(decoder_output.shape))
    print(decoder_output)

    output = rearrange(decoder_output, 't n e -> n t e')
    output =  self.fc(output)
    print("output")
    print("output shape {}".format(output.shape))
    print(output)

    predicted = F.log_softmax(output, dim=-1)
    print("predicted")
    print("predicted shape {}".format(predicted.shape))
    print(predicted)
    # top k
    top_value, top_index = torch.topk(predicted, k=1)
    top_index = torch.squeeze(top_index)
    print("top_index")
    print("top_index shape {}".format(top_index.shape))
    print(top_index)
    print("top_value")
    print("top_value shape {}".format(top_value.shape))
    print(top_value)
    return top_index


Instruction: What should be output size of image classifier model?

Input: I'm performing an image classification task . Images are labeled as 0 1 2. Should be the size of the last linear layer in the model output be 3 or 1 ? In general, for a 3-class operation, the output is set to 3, and as a result of these three, the maximum probability is returned. But I saw that the last layer is set as 1 in some codes. I think it is actually logical. What do you think about ? ( Also I dont use softmax or sigmoid function in last layer.)

Output:
To perform classification into c classes (c = 3 in your example) you need to predict the probability of each class, therefore you need to output a c-dim output.
Usually you do not explicitly apply softmax to the "raw predictions" (aka "logits") - the loss function usually does that for you in a more numerically-robust way (see, e.g., nn.CrossEntropyLoss).
After you trained the model, at inference time you can take argmax over the predicted c logits and output a single scalar - the index of the predicted class. This can only be done during inference since argmax is not a differentiable operation.


Instruction: Pytorch lightning Datamodule override warning: Signature of method '.setup()' does not match signature of base method in class 'LightningDataModule'

Input:
The following is a working Pytorch Lightning DataModule.
import os
from pytorch_lightning import LightningDataModule
import torchvision.datasets as datasets
from torchvision.transforms import transforms
import torch
from torch.utils.data import DataLoader
from Testing.Research.config.paths import mnist_data_download_folder

class PressureDataModule(LightningDataModule):
    def __init__(self, config):
        super().__init__()
        self._config = config

    def prepare_data(self):
        pass

    def setup(self, stage):
        # transform
        transform = transforms.Compose([transforms.ToTensor()])
        mnist_train_full = datasets.MNIST(mnist_data_download_folder, train=True, download=False, transform=self._transforms)
        mnist_test = datasets.MNIST(mnist_data_download_folder, train=False, download=False, transform=self._transforms)

        # train/val split
        train_size = int(self._config.train_size /
                         (self._config.train_size + self._config.val_size) * len(mnist_train_full))
        val_size = len(mnist_train_full) - train_size
        mnist_train, mnist_val = torch.utils.data.random_split(mnist_train_full, [train_size, val_size])

        # assign to use in dataloaders
        self._train_dataset = mnist_train
        self._val_dataset = mnist_val
        self._test_dataset = mnist_test

    def train_dataloader(self):
        return DataLoader(self._train_dataset, batch_size=self._config.batch_size, num_workers=self._config.num_workers)

    def val_dataloader(self):
        return DataLoader(self._val_dataset, batch_size=self._config.batch_size, num_workers=self._config.num_workers)

    def test_dataloader(self):
        return DataLoader(self._test_dataset, batch_size=self._config.batch_size, num_workers=self._config.num_workers)

Pycharm doesn't like setup with

Signature of method 'PressureDataModule.setup()' does not match
signature of base method in class 'LightningDataModule'

If there is no match, why is Pycharm crying?
Is it because of different parameters? What is the correct number of parameters?

What's the correct way to fix this?

Output:
It seems that simple copy-paste the parent method signature solves this issue:
def setup(self, stage: Optional[str] = None) -> None:
    ...


Instruction: How PyTorch model layer weights get initialized implicitly?

Input: I basically defined a model with a Conv2D and linear layer with PyTorch and trained it with a sample dataset. The model seems to run and converge. But I am wondering I did not explicitly initialize the model layer weights (normal or Xavier). Does that mean when I call model.train() before each epoch training, the layer weights will get initialized randomly by default ? If so, how can I explicitly change the type of initializations ?

Output:
The type of initialization depends on the layer. You can check it from the reset_parameters method or from the docs as well.
For both linear and conv layers, it's He initialization (torch.nn.init.kaiming_uniform_).
It's mentioned in the documentation as

The values are initialized from U(−sqrt(k),sqrt(k)).

For embedding layer, it's Normal initialization. (mentioned in docs as N(0,1)).

You can change the type of initialization as mentioned in How to initialize weights in PyTorch?.
conv1 = torch.nn.Conv2d(...)
torch.nn.init.xavier_uniform(conv1.weight)


Instruction: RuntimeError: Given input size: (40x256x1). Calculated output size: (40x253x-2). Output size is too small

Input:
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable
from sklearn import preprocessing

batch_size = 32
num_classes = 8
epochs = 10

img_rows, img_cols = 256, 256
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)

print('x_train shape:', x_train.shape)
print("Training samples: {}".format(x_train.shape[0]))
print("Test samples: {}".format
      (x_test.shape[0]))
x_train = torch.Tensor(x_train).float()
x_test = torch.Tensor(x_test).float()
y_train = torch.LongTensor(y_train)
y_test = torch.LongTensor(y_test)
#Define model
model = nn.Sequential(
        nn.Conv2d(256,80,1, stride=1),
        nn.ReLU(),
        nn.Conv2d(80,40,1, stride=1),
        nn.ReLU(),
        nn.MaxPool2d(4,stride=1),
        nn.Conv2d(40,30,1, stride=1),
        nn.ReLU(),
        nn.Conv2d(30,15,1, stride=1),
        nn.ReLU(),
        nn.Dropout(0.2),
        nn.Flatten(),
        nn.Linear(32, 32),
        nn.Dropout(0.1),
        nn.Linear(num_classes, 256),
        nn.ReLU()
        )
criterion = nn.CrossEntropyLoss()# cross entropy loss

optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

for epoch in range(100):
    optimizer.zero_grad()
    out = model(x_train)
    loss = criterion(out, y_train)
    loss.backward()
    optimizer.step()
    print(f"Epoch {epoch+1}, loss: {loss.item()}")

However when I run this code it produces the error at this specifc line
RuntimeError                              Traceback (most recent call last)
<ipython-input-10-96fa8b09f1ec> in <module>
     63 for epoch in range(100):
     64     optimizer.zero_grad()
---> 65     out = model(x_train)
     66     loss = criterion(out, y_train)
     67     loss.backward()

with the following error message
RuntimeError: Given input size: (40x256x1). Calculated output size: (40x253x-2). Output size is too small
I am unsure on how to fix this as I am new to pytorch and the original model did work in tensorflow.
Any help would be much appreciated

Output:
I'm assuming you are working with images. In that case, there are several issues with your code. Also reading from the comments, there are a couple of things I need to clarify.

I think the most important is the fact you've switched up the axes on the input shape. Unlike in Tensorflow, PyTorch multi-channel maps are shaped with (b, c, h, w) (b: batch size; c: number of channels; hxw: height and width of a feature map.

Also you have defined the first layer as nn.Conv2d(256, 80, 1, stride=1) which means it has 80 filters, and expects a 256-channel input! That's not what you should want, assuming you've following my first point (which is to feed (1,256,256) images) to your model. The number of filters can stay at 80, that's up to you and your neural network design.

A little later down your model, you define a max pool with nn.MaxPool2d(4, stride=1). Just to point out that you are using a kernel size of 4 pixels here. Which means that, at this point, the resulting tensor will have a shape of (b, 40, 253, 253). The change from 256x256 to 253x253 is due to the kernel size being 4.

You flatten after your final convolution into (b, 960135). That's 253*253*15 (feature map dimensions times number of feature maps). Ultimately, the next dense layer needs to have that number as input size. Instead you have put nn.Linear(32, 32) followed by nn.Linear(num_classes, 256). It should rather be something like nn.Linear(253*253*15, n) followed by nn.Linear(n, num_classes). Where n is arbitrarily set and I assumed you wanted an output with num_classes logits at the end.

Also, I read in the comments

If you organize all correctly, I think you have to change the first layer in Sequential() from nn.Conv2d(256,80,1, stride=1) to nn.Conv2d(32,80,1, stride=1)

The batch size doesn't have anything to do with the layer's size parameters. Don't think about the batch axis, it's the first axis. Your model doesn't depend on your batch size!

I would recommend, increasing your kernel sizes, this will allow your feature maps to shrink in dimensions as it goes through the network, at the same time, increasing the number of channels until you hit the flatten layer and fully connected.

Here is your model with some important correction, just enough to make it run:
batch_size = 32
num_classes = 8

img_rows, img_cols = 256, 256
input_shape = (img_rows, img_cols, 1)

# dummy data, would broadcast your data with these shapes
x_train = torch.rand(batch_size, 1, img_rows, img_cols)
y_train = torch.rand(batch_size, 256)

model = nn.Sequential(
    nn.Conv2d(1, 80, 1, stride=1),
    nn.ReLU(),
    nn.Conv2d(80, 40, 1, stride=1),
    nn.ReLU(),
    nn.MaxPool2d(4, stride=1),
    nn.Conv2d(40, 30, 1, stride=1),
    nn.ReLU(),
    nn.Conv2d(30, 15, 1, stride=1),
    nn.ReLU(),
    nn.Dropout(0.2),
    nn.Flatten(),
    nn.Linear(253*253*15, 256),
    nn.Dropout(0.1),
    nn.Linear(256, num_classes),
    nn.ReLU()
)


Instruction: How to replicate PyTorch normalization in OpenCV or NumPy?

Input:
I need to replicate PyTorch image normalization in OpenCV or NumPy.
Quick backstory: I'm doing a project where I'm training in PyTorch but will have to inference in OpenCV due to deploying to an embedded device where I won't have the storage space to install PyTorch.  After training in PyTorch and saving a PyTorch graph I'm then converting to an ONNX graph.  For inferencing in OpenCV I'm opening the image as an OpenCV image (i.e. NumPy array), then resizing, then successively calling cv2.normalize, cv2.dnn.blobFromImage, net.setInput, and net.forward.
I'm getting slightly different accuracy results when test inferencing in PyTorch vs inferencing in OpenCV, and I suspect the difference is due to the normalization process producing a slightly different result between the two.
Here is a quick script I put together to show the difference on a single image.  Note that I'm using grayscale (single-channel) and I'm normalizing into the -1.0 to +1.0 range:
# scratchpad.py

import torch
import torchvision

import cv2
import numpy as np
import PIL
from PIL import Image

TRANSFORM = torchvision.transforms.Compose([
    torchvision.transforms.Resize((224, 224)),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize([0.5], [0.5])
])

def main():
    # 1st show PyTorch normalization

    # open the image as an OpenCV image
    openCvImage = cv2.imread('image.jpg', cv2.IMREAD_GRAYSCALE)
    # convert OpenCV image to PIL image
    pilImage = PIL.Image.fromarray(openCvImage)
    # convert PIL image to a PyTorch tensor
    ptImage = TRANSFORM(pilImage).unsqueeze(0)
    # show the PyTorch tensor info
    print('\nptImage.shape = ' + str(ptImage.shape))
    print('ptImage max = ' + str(torch.max(ptImage)))
    print('ptImage min = ' + str(torch.min(ptImage)))
    print('ptImage avg = ' + str(torch.mean(ptImage)))
    print('ptImage: ')
    print(str(ptImage))

    # 2nd show OpenCV normalization

    # resize the image
    openCvImage = cv2.resize(openCvImage, (224, 224))
    # convert to float 32 (necessary for passing into cv2.dnn.blobFromImage which is not show here)
    openCvImage = openCvImage.astype('float32')
    # use OpenCV version of normalization, could also do this with numpy
    cv2.normalize(openCvImage, openCvImage, 1.0, -1.0, cv2.NORM_MINMAX)
    # show results
    print('\nopenCvImage.shape = ' + str(openCvImage.shape))
    print('openCvImage max = ' + str(np.max(openCvImage)))
    print('openCvImage min = ' + str(np.min(openCvImage)))
    print('openCvImage avg = ' + str(np.mean(openCvImage)))
    print('openCvImage: ')
    print(str(openCvImage))

    print('\ndone !!\n')
# end function

if __name__ == '__main__':
    main()

Here is the test image that I'm using:

and here are the results I'm getting currently:
$ python3 scratchpad.py

ptImage.shape = torch.Size([1, 1, 224, 224])
ptImage max = tensor(0.9608)
ptImage min = tensor(-0.9686)
ptImage avg = tensor(0.1096)
ptImage:
tensor([[[[ 0.0431, -0.0431,  0.1294,  ...,  0.8510,  0.8588,  0.8588],
          [ 0.0510, -0.0510,  0.0980,  ...,  0.8353,  0.8510,  0.8431],
          [ 0.0588, -0.0431,  0.0745,  ...,  0.8510,  0.8588,  0.8588],
          ...,
          [ 0.6157,  0.6471,  0.5608,  ...,  0.6941,  0.6627,  0.6392],
          [ 0.4902,  0.3961,  0.3882,  ...,  0.6627,  0.6471,  0.6706],
          [ 0.3725,  0.4039,  0.5451,  ...,  0.6549,  0.6863,  0.6549]]]])

openCvImage.shape = (224, 224)
openCvImage max = 1.0000001
openCvImage min = -1.0
openCvImage avg = 0.108263366
openCvImage:
[[ 0.13725497 -0.06666661  0.20000008 ...  0.8509805   0.8666668
   0.8509805 ]
 [ 0.15294124 -0.06666661  0.09019614 ...  0.8274511   0.8431374
   0.8274511 ]
 [ 0.12156869 -0.06666661  0.0196079  ...  0.8509805   0.85882366
   0.85882366]
 ...
 [ 0.5843138   0.74117655  0.5450981  ...  0.83529425  0.59215695
   0.5764707 ]
 [ 0.6862746   0.34117654  0.39607853 ...  0.67843145  0.6705883
   0.6470589 ]
 [ 0.34117654  0.4117648   0.5215687  ...  0.5607844   0.74117655
   0.59215695]]

done !!

As you can see the results are similar but definitely not exactly the same.
How can I do the normalization in OpenCV and have it come out exactly or almost exactly the same as the PyTorch normalization?  I've tried various options in both OpenCV and with NumPy but could not get it any closer than the above results, which are substantially different.
-- Edit ---------------------------
In response to Ivan, I also tried this:
# resize the image
openCvImage = cv2.resize(openCvImage, (224, 224))
# convert to float 32 (necessary for passing into cv2.dnn.blobFromImage which is not show here)
openCvImage = openCvImage.astype('float32')
mean = np.mean(openCvImage)
stdDev = np.std(openCvImage)
openCvImage = (openCvImage - mean) / stdDev
# show results
print('\nopenCvImage.shape = ' + str(openCvImage.shape))
print('openCvImage max = ' + str(np.max(openCvImage)))
print('openCvImage min = ' + str(np.min(openCvImage)))
print('openCvImage avg = ' + str(np.mean(openCvImage)))
print('openCvImage: ')
print(str(openCvImage))

Which results in:
openCvImage.shape = (224, 224)
openCvImage max = 2.1724665
openCvImage min = -2.6999729
openCvImage avg = 7.298528e-09
openCvImage:
[[ 0.07062991 -0.42616782  0.22349077 ...  1.809422    1.8476373
   1.809422  ]
 [ 0.10884511 -0.42616782 -0.04401573 ...  1.7520993   1.7903144
   1.7520993 ]
 [ 0.0324147  -0.42616782 -0.21598418 ...  1.809422    1.8285296
   1.8285296 ]
 ...
 [ 1.1597633   1.5419154   1.0642253  ...  1.7712069   1.178871
   1.1406558 ]
 [ 1.4081622   0.56742764  0.70118093 ...  1.3890547   1.3699471
   1.3126242 ]
 [ 0.56742764  0.7393961   1.0069026  ...  1.1024406   1.5419154
   1.178871  ]]

Which is similar to the PyTorch normalization but clearly not the same.
I'm attempting to achieve normalization in OpenCV that produces the same result as the PyTorch normalization.
I realize that due to slight differences in the resizing operation (and possibly very small rounding differences) I'll probably never get exactly the same normalized result but I'd like to get as close as possible to the PyTorch result.

Output:
This probably would be helpful
If you look at actual implementation of
torchvision.transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225],
    )

Below block is what it actually does:
import numpy as np
from PIL import Image
MEAN = 255 * np.array([0.485, 0.456, 0.406])
STD = 255 * np.array([0.229, 0.224, 0.225])
img_pil = Image.open("ty.jpg")
x = np.array(img_pil)
x = x.transpose(-1, 0, 1)
x = (x - MEAN[:, None, None]) / STD[:, None, None]

Here i have done it on image


Instruction: what is Pytorch's add_module()?

Input:
I stumbled upon the method add_module() in a Pytorch model.
The doc only states

Adds a child module to the current module.
The module can be accessed as an attribute using the given name.

I don't understand what "adding a child module" means.
How is it different from just setting a pointer to the other module using self._other module = other_module?
What are the nuances?

Output:
As mentioned here: https://discuss.pytorch.org/t/when-to-use-add-module-function/10534
In general, you won’t need to call add_module. One potential use case is the following:
class Net(nn.Module):
    def __init__(self):
    super(Net, self).__init__()
        modules = [...]  # some list of modules
        for module in modules:
            self.add_module(...)


Instruction: How to merge two torch.utils.data dataloaders with a single operation

Input:
I have two dataloaders and I would like to merge them without redefining the datasets, in my case train_dataset and val_dataset.
train_loader = DataLoader(train_dataset, batch_size = 512, drop_last=True,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size = 512, drop_last=False)

Wanted result:
train_loader = train_loader + val_loader

Output:
Data loaders are iterators, you can implement a function that returns an iterator which yields the dataloaders' content, one dataloader after the other.
Given a number of iterators itrs, it would iterate over each iterator and in turn iterate over each iterator yielding one batch at a time. A possible implementation would be as simple as:
def itr_merge(*itrs):
    for itr in itrs:
        for v in itr:
            yield v

Here is an usage example:
>>> dl1 = DataLoader(TensorDataset(torch.zeros(5, 1)), batch_size=2, drop_last=True)
>>> dl2 = DataLoader(TensorDataset(torch.ones(10, 1)), batch_size=2)

>>> for x in itr_merge(dl1, dl2):
>>>   print(x)
[tensor([[0.], [0.]])]
[tensor([[0.], [0.]])]
[tensor([[1.], [1.]])]
[tensor([[1.], [1.]])]
[tensor([[1.], [1.]])]
[tensor([[1.], [1.]])]
[tensor([[1.], [1.]])]


Instruction: Input batch size doesn't match target batch size in CrossEntropyLoss function

Input:
I've been trying to build a model from scratch to recognize handwritten digits from the MNIST dataset with the help of PyTorch and the DataLoader class from FastAI. So far, I've been using a linear model that has 784 inputs (a flattened grayscale 28 by 28 handwritten digit image tensor) and 10 outputs.
simple_linear = torch.nn.Linear(784, 10)

My training data is organized as such:
train_x = torch.cat([stacked_zeros, stacked_ones, stacked_twos, stacked_threes,
                     stacked_fours, stacked_fives, stacked_sixes, stacked_sevens,
                     stacked_eights, stacked_nines]).view(-1, 28*28)

train_y = torch.nn.functional.one_hot(tensor([0] * len(zeros) + [1] * len(ones) + [2] * len(twos) +
                 [3] * len(threes) + [4] * len(fours) + [5] * len(fives) +
                 [6] * len(sixes) + [7] * len(sevens) + [8] * len(eights) +
                 [9] * len(nines)).unsqueeze(1))

My x variables have shape [784] while y variables are labeled using one-hot encoded vectors with [1, 10] shape.
The loss function I chose based on research is torch.nn.CrossEntropyLoss and the following code gives me an error:
mnist_loss = torch.nn.CrossEntropyLoss()
mnist_loss(simple_linear(train_x[0]), train_y[0])

ValueError                                Traceback (most recent call last)
<ipython-input-245-03f54a6a43fb> in <module>()
----> 1 tst(simple_linear(x), y)

8 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2260     if input.size(0) != target.size(0):
   2261         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'
-> 2262                          .format(input.size(0), target.size(0)))
   2263     if dim == 2:
   2264         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (1) to match target batch_size (10).

I've tried reshaping my x variables and y variables but I always get a similar error. How must my data be structured in order for the loss function to work?

Output:
The torch.nn.CrossEntropyLoss function doesn't take targets as one-hot-encodings!
Just pass the label index, so basically:
train_y = torch.tensor([0] * len(zeros) + [1] * len(ones) + [2] * len(twos) +
                 [3] * len(threes) + [4] * len(fours) + [5] * len(fives) +
                 [6] * len(sixes) + [7] * len(sevens) + [8] * len(eights) +
                 [9] * len(nines)).unsqueeze(1)

Here's a suggestion, you could write everything like:
dataset = [stacked_zeros, stacked_ones, stacked_twos, stacked_threes,
           stacked_fours, stacked_fives, stacked_sixes, stacked_sevens,
           stacked_eights, stacked_nines]

train_x = torch.cat(dataset)
train_y = torch.tensor([[i]*d.size(0) for i, d in enumerate(dataset)])


Instruction: Loading the pre-trained model of torch and sentence_transformers when running in a docker container failing

Input:
I am getting below error while loading the pre-trained model of torch and sentence_transformers("distilbert-base-nli-stsb-mean-tokens") when trying to run in a docker container.
Error: Invalid value for '-A' / '--app':
 Unable to load celery application.
 While trying to load the module app.celery the following error occurred:
 Traceback (most recent call last):
   File "/usr/local/lib/python3.8/site-packages/celery/bin/celery.py", line 53, in convert
     return find_app(value)
   File "/usr/local/lib/python3.8/site-packages/celery/app/utils.py", line 384, in find_app
     sym = symbol_by_name(app, imp=imp)
   File "/usr/local/lib/python3.8/site-packages/kombu/utils/imports.py", line 56, in symbol_by_name
     module = imp(module_name, package=package, **kwargs)
   File "/usr/local/lib/python3.8/site-packages/celery/utils/imports.py", line 100, in import_from_cwd
     return imp(module, package=package)
   File "/usr/local/lib/python3.8/importlib/__init__.py", line 127, in import_module
     return _bootstrap._gcd_import(name[level:], package, level)
   File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
   File "<frozen importlib._bootstrap>", line 991, in _find_and_load
   File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
   File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
   File "<frozen importlib._bootstrap_external>", line 783, in exec_module
   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
   File "/code/app.py", line 997, in <module>
     load_model()
   File "/code/app.py", line 255, in load_model
     embedder = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')
   File "/usr/local/lib/python3.8/site-packages/sentence_transformers/SentenceTransformer.py", line 48, in __init__
     os.makedirs(model_path, exist_ok=True)
   File "/usr/local/lib/python3.8/os.py", line 213, in makedirs
     makedirs(head, exist_ok=exist_ok)
   File "/usr/local/lib/python3.8/os.py", line 213, in makedirs
     makedirs(head, exist_ok=exist_ok)
   File "/usr/local/lib/python3.8/os.py", line 213, in makedirs
     makedirs(head, exist_ok=exist_ok)
   [Previous line repeated 1 more time]
   File "/usr/local/lib/python3.8/os.py", line 223, in makedirs
     mkdir(name, mode)
 PermissionError: [Errno 13] Permission denied: '/nonexistent'

Here it is saying permission denied error while creating the folder. But I have tried providing USER root in the Dockerfile. Stuck with this issue for long time. Please anyone help me here.
Updated:
My Dockerfile:
FROM python:3.8.5-slim

WORKDIR /code

ENV ENVIRONMENT='LOCAL'
ENV FLASK_APP=app.py
ENV FLASK_RUN_HOST=0.0.0.0
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1

RUN apt-get update && apt-get install -y sudo netcat apt-utils
RUN apt-get install -y python3-dev  build-essential python3-pip

COPY ./requirements_local.txt /code/requirements_local.txt
RUN pip install -r /code/requirements_local.txt

EXPOSE 8000
COPY . /code/

CMD [ "gunicorn", "app:app", "-b", "0.0.0.0:8000","--timeout","7200"]

Docker-compose:
services:
  web:
    build:
      context: .
      dockerfile: ./Dockerfile.prod
    hostname: flaskapp
    env_file:
      - ./.env.prod
    links:
      - redis
      - celery
    depends_on:
      - redis
    volumes:
      - data:/code
      - type: bind
        source: /home/ubuntu/models
        target: /mnt/models

Output:
sentence-transformers downloads and stores model in ~/.cache directory (or whatever the cache_folder evaluates to be in - https://github.com/UKPLab/sentence-transformers/blob/a13a4ec98b8fdda83855aca7992ea793444a207f/sentence_transformers/SentenceTransformer.py#L63). For you this looks like the /nonexistant directory. The permission denied error suggests you do not have permission to access that directory (to create cache folder).
You can modify the Dockerfile to create this directory and make it accessible to any user that needs to access this:-
RUN mkdir ~/.cache
RUN chmod -R 777 ~/.cache # don't do this in production - modify command to give permission to users who require it.

Or you could try downloading the model in the Dockerfile itself -
RUN python -c 'from sentence-transformers import SentenceTransformer; embedder = SentenceTransformer("distilbert-base-nli-stsb-mean-tokens")'


Instruction: Problems identifying images in pytorch

Input:
I am really new at deep learning and I am studying how to properly run neural networks using pytorch. Currently I am trying to read a dataset of images using the following code:
from torch.utils.data import Dataset, DataLoader
from torchvision import datasets, transforms
from torchvision import transforms, utils

transformations = transforms.Compose([
    transforms.Resize(255),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
train_set = datasets.ImageFolder('faces/train', transform = transformations)
test_set = datasets.ImageFolder('faces/test', transform = transformations)

train_loader = DataLoader(train_set, batch_size=60, shuffle = True)
test_loader = DataLoader(test_set, batch_size=60, shuffle = True)

once done that, I am trying to read the images to start running the neural networks on the images, first by separating the imgs from the labels using:
img, labels = next(iter(train_loader))

And then I received the following message:
UnidentifiedImageError: cannot identify image file <_io.BufferedReader name='faces/train/karyadi/karyadi_straight_angry_open.pgm'>

The structure of the folders on my working directory where I have stored the images is as follows:

faces:

Train: 10 folders, each one with around 90 images in pgm format.
Test: 1o folders, each one with around 90 images in pgm format.

does anyone know what the problem might be?
thanks in advance.

Output:
I finally found the problem. This post was very useful. The gist of it is that PIL has problems importing images of certain sizes (I do not have all the details about this). At the end I use cv2 to import the pgm images one by one and then convert them into 32-bit ndarrays to export them in jpeg format. Here is my code for one image:
import cv2
img = cv2.imread('faces/train/karyadi/karyadi_straight_angry_open.pgm')
img = np.full(img.shape, img, dtype=np.float32)
cv2.imwrite('img.jpeg', img)


Instruction: Batch normalization makes training worse

Input:
I am trying to implement the batch normalization with Pytorch and use a simple fully connected neural network to approximate a given function.
The code is as follows. The result shows that the neural network without the batch normalization performs better than that with the batch normalization technique. This means that the batch normalization makes the training even worse. Could someone explain this result? Thanks!
import matplotlib.pyplot as plt
import numpy as np
import torch

class Net(torch.nn.Module):

    def __init__(self, num_inputs, num_outputs, hidden_size=256, is_bn=True):
        super(Net, self).__init__()
        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.is_bn = is_bn

        # no bias is needed if batch normalization
        if self.is_bn:
            self.linear1 = torch.nn.Linear(num_inputs, hidden_size, bias=False)
            self.linear2 = torch.nn.Linear(hidden_size, hidden_size, bias=False)
        else:
            self.linear1 = torch.nn.Linear(num_inputs, hidden_size)
            self.linear2 = torch.nn.Linear(hidden_size, hidden_size)

        self.linear3 = torch.nn.Linear(hidden_size, num_outputs)

        if self.is_bn:
            self.bn1 = torch.nn.BatchNorm1d(hidden_size)
            self.bn2 = torch.nn.BatchNorm1d(hidden_size)

        self.activation = torch.nn.ReLU()

    def forward(self, inputs):
        x = inputs
        if self.is_bn:
            x = self.activation(self.bn1(self.linear1(x)))
            x = self.activation(self.bn2(self.linear2(x)))
        else:
            x = self.activation(self.linear1(x))
            x = self.activation(self.linear2(x))
        out = self.linear3(x)
        return out

torch.manual_seed(0)    # reproducible

Nx = 100
x = torch.linspace(-1., 1., Nx)
x = torch.reshape(x, (Nx, 1))
y = torch.sin(3*x)

fcn_bn, fcn_no_bn = Net(num_inputs=1, num_outputs=1, is_bn=True), Net(num_inputs=1, num_outputs=1, is_bn=False)

criterion = torch.nn.MSELoss()
optimizer_bn = torch.optim.Adam(fcn_bn.parameters(), lr=0.001)
optimizer_no_bn = torch.optim.Adam(fcn_no_bn.parameters(), lr=0.001)

total_epoch = 5000

# record loss history
loss_history_bn = np.zeros(total_epoch)
loss_history_no_bn = np.zeros(total_epoch)

fcn_bn.train()
fcn_no_bn.train()
for epoch in range(total_epoch):

    optimizer_bn.zero_grad()
    loss = criterion(fcn_bn(x), y)
    loss_history_bn[epoch] = loss.item()
    loss.backward()
    optimizer_bn.step()

    optimizer_no_bn.zero_grad()
    loss = criterion(fcn_no_bn(x), y)
    loss_history_no_bn[epoch] = loss.item()
    loss.backward()
    optimizer_no_bn.step()

    if epoch%1000 == 0:
        print("epoch: %d; MSE (with bn): %.2e; MSE (without bn): %.2e"%(epoch, loss_history_bn[epoch], loss_history_no_bn[epoch]))

fcn_bn.eval()
fcn_no_bn.eval()

plt.figure()
plt.semilogy(np.arange(total_epoch), loss_history_bn, label='neural network (with bn)')
plt.semilogy(np.arange(total_epoch), loss_history_no_bn, label='neural network (without bn)')
plt.legend()

plt.figure()
plt.plot(x, y, '-', label='exact')
plt.plot(x, fcn_bn(x).detach(), 'o', markersize=2, label='neural network (with bn)')
plt.plot(x, fcn_no_bn(x).detach(), 'o', markersize=2, label='neural network (without bn)')
plt.legend()

plt.figure()
plt.plot(x, np.abs(fcn_bn(x).detach() - y), 'o', markersize=2, label='neural network (with bn)')
plt.plot(x, np.abs(fcn_no_bn(x).detach() - y), 'o', markersize=2, label='neural network (without bn)')
plt.legend()

plt.show()

The result is as follows:
epoch: 0; MSE (with bn): 3.99e-01; MSE (without bn): 4.84e-01
epoch: 1000; MSE (with bn): 4.70e-05; MSE (without bn): 1.27e-06
epoch: 2000; MSE (with bn): 1.81e-04; MSE (without bn): 7.93e-07
epoch: 3000; MSE (with bn): 2.73e-04; MSE (without bn): 7.45e-07
epoch: 4000; MSE (with bn): 4.04e-04; MSE (without bn): 5.68e-07

Output:
To provide an alternate view to the answer that Khalid linked in the comments, which puts a stronger focus on generalization performance rather than training loss, consider this:
Batch Normalization has been postulated to have a regularizing effect. Luo et al. look at BN as a decomposition into population normalization and gamma decay and observe similar training loss curves as you do (comparing BN to no BN - note, however, that they use vanilla SGD and not Adam). There are a couple of things that affect BN (as outlined also in Khalid's link): The batch size, for example, on the one hand should be large enough for robust estimation of population parameters, however, with increasing size of the batch generalization performance can also drop (see Luo et al.'s paper: the gist is that lower batch sizes result in noisy population parameter estimates, essentially perturbing the input).
In your case I would not intuitively have expected a big difference (given how your data is set up), but maybe someone deeper into the theoretical analysis of BN can still provide insights.


Instruction: What's the best way of handling the GAN training output?

Input:
Supervising the training of GANs usually involves outputting not only metrics, but also images at a certain interval of epochs. My application also involves printing tables. I use jupyter notebooks, but just printing it all on the notebooks makes each notebook for each experiment way too large (+100 MB), and the internet browser gets slow and crashes often because of that.
I suppose the usual practice would be to save the image outputs somewhere else (either with tensorboard or just plain image files), but that would not be ideal for me because I like to observe each image together with the text/table output relative to it's epoch. It would be even better if I could save the entire training output into a single file, so I could just scroll down through it, observing each epoch output with text/table/image, just like the jupyter notebook output. Is there a way of implementing this? Or perhaps a better way that I'm not considering?
Thank you and sorry if this question is somehow inadequate. If so, let me know and I'll delete it.

Output:
Well It's a matter of preference, how someone likes to have it. When I trained a GAN, I handled it in the following way, for the loss and other such values I printed them simply on the notebook per epoch manner as we would do with any other models along with that I would generate an image made by the generator and save it to a folder to check If the generator is getting better. But to make it easier for me to observe those images, I've used meaningful labeling for those images, I've included epoch number in the label, which helps to go through the images pretty quickly. However going through the images would be a bit tedious job, so to make it a bit more comforting I've merged all those pictures produced at every epoch (let's say 500) and made a video, that's pretty cool to watch. I found this way of training my GAN and producing its output really easier but as I said It would be a matter of preference.
Here's the code-
https://github.com/khalidsaifullaah/Classic-Deep-Learning-Models/tree/master/GAN


Instruction: Validation and training loss per batch and epoch

Input:
I am using Pytorch to run some deep learning models. I am currently keeping track of training and validation loss per epoch, which is pretty standard. However, what is the best way of going about keeping track of training and validation loss per batch/iteration?
For training loss, I could just keep a list of the loss after each training loop. But, validation loss is calculated after a whole epoch, so I’m not sure how to go about the validation loss per batch. The only thing I can think of is to run the whole validation step after each training batch and keeping track of those, but that seems overkill and a lot of computation.
For example, the training is like this:
for epoch in range(2):  # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
    # get the inputs; data is a list of [inputs, labels]
    inputs, labels = data

    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = net(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    # print statistics
    running_loss += loss.item()

And for validation loss:
with torch.no_grad():
    for data in testloader:
        images, labels = data
        outputs = net(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()
        # validation loss
        batch_loss = error(outputs.float(), labels.long()).item()
        loss_test += batch_loss
    loss_test /= len(testloader)

The validation loss/test part is done per epoch. I’m looking for a way to get the validation loss per batch, which is my point above.
Any tips?

Output:
Well, you're right that's the way to do it "run the whole validation step after each training batch and keeping track of those" and also as you've thought it's pretty time-consuming and would be overkill. However, If that's something you really need then there's a way you can do it. What you can do is, let's say you've 1000  batches in your data. Now to calculate per batch val_loss you can choose not to run the validation step for each of the batch (then you'd have to do it 1000 times!) but for a small subset of those batches, let's say 50/100 (choose as you please or find feasible). Now, you can use some statistical power so that your calculation for 50/100 batches becomes very very close to that of 1000 batches (meaning this val_loss for a small number of batches must be as close as to those of 1000 batches if you had calculated that), so to achieve it you can introduce some randomness in your batch selection.
This means you randomly select 100 batches from your 1000 batches for which you'll run the validation step.


Instruction: pytorch runs slow when data are pre-transported to GPU

Input:
I have a model written in pytorch. Since my dataset is small, I can directly load all of the data to GPU. However, I found the forward speed becomes slow if I do so. The following is a runnable example. Specifically, I have the model:
import numpy as np
from time import time
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader

def knn(x, k):
    inner = -2*torch.matmul(x.transpose(2, 1), x)
    xx = torch.sum(x**2, dim=1, keepdim=True)
    pairwise_distance = -xx - inner - xx.transpose(2, 1)
    idx = pairwise_distance.topk(k=k, dim=-1)[1]   # (batch_size, num_points, k)
    return idx

def get_graph_feature(x, k=20, idx=None):
    batch_size = x.size(0)
    num_points = x.size(2)
    x = x.view(batch_size, -1, num_points)
    if idx is None:
        idx = knn(x, k=k)   # (batch_size, num_points, k)
    idx_base = torch.arange(0, batch_size, device=x.device).view(-1, 1, 1)*num_points
    idx = idx + idx_base
    idx = idx.view(-1)
    _, num_dims, _ = x.size()
    x = x.transpose(2, 1).contiguous()   # (batch_size, num_points, num_dims)  -> (batch_size*num_points, num_dims) #   batch_size * num_points * k + range(0, batch_size*num_points)
    feature = x.view(batch_size*num_points, -1)[idx, :]
    feature = feature.view(batch_size, num_points, k, num_dims)
    x = x.view(batch_size, num_points, 1, num_dims).repeat(1, 1, k, 1)
    feature = torch.cat((feature-x, x), dim=3).permute(0, 3, 1, 2).contiguous()
    return feature

class DGCNN(nn.Module):
    def __init__(self, k=25, output_channels=10):
        super(DGCNN, self).__init__()
        self.k = k
        self.bn1 = nn.BatchNorm2d(64)
        self.bn2 = nn.BatchNorm2d(64)
        self.bn3 = nn.BatchNorm2d(128)
        self.bn4 = nn.BatchNorm2d(256)
        self.bn5 = nn.BatchNorm1d(1024)
        self.conv1 = nn.Sequential(nn.Conv2d(6, 64, kernel_size=1, bias=False),
                                   self.bn1,
                                   nn.LeakyReLU(negative_slope=0.2))
        self.conv2 = nn.Sequential(nn.Conv2d(64*2, 64, kernel_size=1, bias=False),
                                   self.bn2,
                                   nn.LeakyReLU(negative_slope=0.2))
        self.conv3 = nn.Sequential(nn.Conv2d(64*2, 128, kernel_size=1, bias=False),
                                   self.bn3,
                                   nn.LeakyReLU(negative_slope=0.2))
        self.conv4 = nn.Sequential(nn.Conv2d(128*2, 256, kernel_size=1, bias=False),
                                   self.bn4,
                                   nn.LeakyReLU(negative_slope=0.2))
        self.conv5 = nn.Sequential(nn.Conv1d(512, 1024, kernel_size=1, bias=False),
                                   self.bn5,
                                   nn.LeakyReLU(negative_slope=0.2))
        self.linear1 = nn.Linear(1024*2, 512, bias=False)
        self.bn6 = nn.BatchNorm1d(512)
        self.dp1 = nn.Dropout()
        self.linear2 = nn.Linear(512, 256)
        self.bn7 = nn.BatchNorm1d(256)
        self.dp2 = nn.Dropout()
        self.linear3 = nn.Linear(256, output_channels)

    def forward(self, x):
        x = x.transpose(2, 1)
        batch_size = x.size(0)
        x = get_graph_feature(x, k=self.k)
        x = self.conv1(x)
        x1 = x.max(dim=-1, keepdim=False)[0]
        x = get_graph_feature(x1, k=self.k)
        x = self.conv2(x)
        x2 = x.max(dim=-1, keepdim=False)[0]
        x = get_graph_feature(x2, k=self.k)
        x = self.conv3(x)
        x3 = x.max(dim=-1, keepdim=False)[0]
        x = get_graph_feature(x3, k=self.k)
        x = self.conv4(x)
        x4 = x.max(dim=-1, keepdim=False)[0]
        x = torch.cat((x1, x2, x3, x4), dim=1)
        x = self.conv5(x)
        x1 = F.adaptive_max_pool1d(x, 1).view(batch_size, -1)
        x2 = F.adaptive_avg_pool1d(x, 1).view(batch_size, -1)
        x = torch.cat((x1, x2), 1)
        x = F.leaky_relu(self.bn6(self.linear1(x)), negative_slope=0.2)
        x = self.dp1(x)
        x = F.leaky_relu(self.bn7(self.linear2(x)), negative_slope=0.2)
        x = self.dp2(x)
        x = self.linear3(x)
        return x

Here is what the dataloader and test function looks like:
class my_loader(Dataset):
    def __init__(self, device):
        self.data = torch.rand(256, 2048, 3).to(device).float()
        self.labels = torch.rand(256).to(device).long()

    def __getitem__(self, ind):
        return self.data[ind], self.labels[ind]

    def __len__(self):
        return len(self.data)

def test():
    device = torch.device('cuda:2')
    test_set = my_loader(device)
    test_loader = DataLoader(test_set, batch_size=16, shuffle=True, num_workers=0)
    model = DGCNN().to(device)
    model.eval()

    #---------- this one is 0.12s --------------#
    for inputs, labels in test_loader:
        tic = time()
        pred = model(inputs)
        print('time1 {}'.format(time() - tic))
    print('------------------')

    #---------- this one is 0.004s --------------#
    for inputs, labels in test_loader:
        inputs = inputs.detach().cpu().to(device)
        tic = time()
        pred = model(inputs)
        print('time2 {}'.format(time() - tic))
    print('------------------')

    #---------- this one is 0.12s --------------#
    for inputs, labels in test_loader:
        tic = time()
        inputs = inputs.detach().cpu().to(device)
        pred = model(inputs)
        print('time3 {}'.format(time() - tic))
    print('------------------')

Basically, it seems that if there is no explicit call of gpu to cpu transportation either before or after the forward propagation, the forward propagation would cost more time. It just seems like that the forward propagation is implicitly doing gpu->cpu transportation.

Output:
I played around with the code a little bit, and I think the problem is that you are measuring times for both cases in the same run. Here is my boiled down version of your code since your model crushed my GPU memory:
class DGCNN(nn.Module):
    def __init__(self, num_layers):
        super(DGCNN, self).__init__()
        self.layers = nn.ModuleList([nn.Linear(256, 256) for _ in range(1200)])

    def forward(self, x):
        x = x.view(-1, 256)
        for layer in self.layers:
            x = layer(x)
        return x

class my_loader(Dataset):
    def __init__(self, device):
        self.data = torch.rand(256, 2048, 3).to(device).float()
        self.labels = torch.rand(256).to(device).long()

    def __getitem__(self, ind):
        return self.data[ind], self.labels[ind]

    def __len__(self):
        return len(self.data)

Now, here I demonstrate different versions of test().
Version #1:
def test():
    device = torch.device('cuda:0')
    test_set = my_loader(device)
    test_loader = DataLoader(test_set, batch_size=16, shuffle=True, num_workers=0)
    model = DGCNN().to(device)
    model.eval()

    #---------- this one is 0.12s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs)
    tac = time()
    print(f'# First case -> Full forward pass: {tac - tic:.6f}')

    #---------- this one is 0.004s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs.detach().cpu().to(device))
    tac = time()
    print(f'# Second case -> Full forward pass: {tac - tic:.6f}')

>>> # First case -> Full forward pass: 3.105103, # Second case -> Full forward pass: 2.831652

Now I switched the order of timing calculations for the cases. Version #2:
def test():
    device = torch.device('cuda:0')
    test_set = my_loader(device)
    test_loader = DataLoader(test_set, batch_size=16, shuffle=True, num_workers=0)
    model = DGCNN().to(device)
    model.eval()

    #---------- this one is 0.004s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs.detach().cpu().to(device))
    tac = time()
    print(f'# Second case -> Full forward pass: {tac - tic:.6f}')

    #---------- this one is 0.12s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs)
    tac = time()
    print(f'# First case -> Full forward pass: {tac - tic:.6f}')

>>> # Second case -> Full forward pass: 3.288522, # First case -> Full forward pass: 2.583231

Apparently, the first timing you calculate seems to end up slower. So, I calculated these timings separately in different runs with fresh kernels. Version #3:
def test():
    device = torch.device('cuda:0')
    test_set = my_loader(device)
    test_loader = DataLoader(test_set, batch_size=16, shuffle=True, num_workers=0)
    model = DGCNN().to(device)
    model.eval()

    #---------- this one is 0.12s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs)
    tac = time()
    print(f'# First case -> Full forward pass: {tac - tic:.6f}')

>>> # First case -> Full forward pass: 3.091592

Version #4:
def test():
    device = torch.device('cuda:0')
    test_set = my_loader(device)
    test_loader = DataLoader(test_set, batch_size=16, shuffle=True, num_workers=0)
    model = DGCNN().to(device)
    model.eval()

    #---------- this one is 0.004s --------------#
    tic = time()
    for inputs, labels in test_loader:
        pred = model(inputs.detach().cpu().to(device))
    tac = time()
    print(f'# Second case -> Full forward pass: {tac - tic:.6f}')

>>> # Second case -> Full forward pass: 3.190248

So, by testing one at a time, it seems like pred = model(inputs) runs slightly faster than pred = model(inputs.detach().cpu().to(device)), which is the obvious expected result.


Instruction: Getting an error while training Resnet50 on Imagenet at 14th Epoch

Input:
I am training Resnet50 on imagenet using the script provided from PyTorch (with a slight trivial tweak for my purpose). However, I am getting the following error after 14 epochs of training. I have allocated 4 gpus in the server I'm using to run this. Any pointers as to what this error is about would be appreciated. Thanks a lot!
Epoch: [14][5000/5005]  Time 1.910 (2.018)  Data 0.000 (0.191)  Loss 2.6954 (2.7783)    Total 2.6954 (2.7783)   Reg 0.0000  Prec@1 42.969 (40.556)  Prec@5 64.844 (65.368)
Test: [0/196]   Time 86.722 (86.722)    Loss 1.9551 (1.9551)    Prec@1 51.562 (51.562)  Prec@5 81.641 (81.641)
Traceback (most recent call last):
  File "main_group.py", line 549, in <module>
  File "main_group.py", line 256, in main

  File "main_group.py", line 466, in validate
    if args.gpu is not None:
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 801, in __next__
    return self._process_data(data)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 846, in _process_data
    data.reraise()
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/_utils.py", line 385, in reraise
    raise self.exc_type(msg)
OSError: Caught OSError in DataLoader worker process 11.
Original Traceback (most recent call last):
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line 178, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 138, in __getitem__
    sample = self.loader(path)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 174, in default_loader
    return pil_loader(path)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 155, in pil_loader
    with open(path, 'rb') as f:
OSError: [Errno 5] Input/output error: '/data/users2/oiler/github/imagenet-data/val/n02102973/ILSVRC2012_val_00009130.JPEG'

Output:
It is difficult to tell what the problem is just by looking at the error you have posted.
All we know is that there was an issue reading the file at '/data/users2/oiler/github/imagenet-data/val/n02102973/ILSVRC2012_val_00009130.JPEG'.
Try the following:

Confirm the file actually exists.
Confirm that it is infact a valid JPEG and not corrupted (by viewing it).
Confirm that you can open it with Python and also load it manually with PIL.
If none of that works, try deleting the file. Do you get the same error on another file in the folder?


Instruction: Difference between CrossEntropyLoss and NNLLoss with log_softmax in PyTorch?

Input:
When I am building a classifier in PyTorch, I have 2 options to do

Using the nn.CrossEntropyLoss without any modification in the model
Using the nn.NNLLoss with F.log_softmax added as the last layer in the model

So there are two approaches.
Now, what approach should anyone use, and why?

Output:
They're the same.
If you check the implementation, you will find that it calls nll_loss after applying log_softmax on the incoming arguments.
return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)

Edit: seems like the links are now broken, here's the C++ implementation which shows the same information.