Instruction: Batch size reduces accuracy of ensemble of pretrained CNNs

Input:
I'm trying to implement basic softmax-based voting, which I take a couple of pretrained CNNs, softmax their outputs, add them together and then use argmax as final output.
So I loaded 4 different pretrained CNNs (vgg11, vgg13, vgg16, vgg19) from "chenyaofo/pytorch-cifar-models" that were trained on CIFAR10 -- I didn't train them.

When I iterate over the test set with DataLoader with batch_size=128/256,  I get to 94% accuracy;

When I iterate over the test set with batch_size=1, I get to 69% accuracy.

How could it be?
This is the code:
import torch
from tqdm import tqdm
from torchvision import datasets, transforms, models
from torch.utils.data import DataLoader
import torch.nn as nn
import torch

torch.cuda.empty_cache()

model_names = [
        "cifar10_vgg11_bn",
        "cifar10_vgg13_bn",
        "cifar10_vgg16_bn",
        "cifar10_vgg19_bn",
        # "cifar10_resnet56",
]

batch_size = 2

test_transform = transforms.Compose([
                    transforms.ToTensor(),
])

def load_models():
    models = []
    for model_name in model_names:
        model = torch.hub.load("chenyaofo/pytorch-cifar-models", model_name, pretrained=True)
        models.append(model)
    return models

testset = datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=test_transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,
                                         shuffle=False)

import torch.nn as nn
import torch

class MyEnsemble(nn.Module):

    def __init__(self, modelA, modelB, modelC, modelD):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        self.modelC = modelC
        self.modelD = modelD
        # self.modelE = modelE

    def forward(self, x):
        out1 = self.modelA(x)
        out2 = self.modelB(x)
        out3 = self.modelC(x)
        out4 = self.modelD(x)
        # out5 = self.modelE(x)

        # print(out1.shape)

        out1 = torch.softmax(out1, dim=1)
        out2 = torch.softmax(out2, dim=1)
        out3 = torch.softmax(out3, dim=1)
        out4 = torch.softmax(out4, dim=1)

        out = out1 + out2 + out3 + out4

        return out

from EnsembleModule import MyEnsemble
from data import load_models, testloader
import torch
from tqdm import tqdm

device = 'cuda' if torch.cuda.is_available() else 'cpu'

models = load_models()

model = MyEnsemble(models[0], models[1], models[2], models[3])

model.to(device)

total = 0
correct = 0
with torch.no_grad():
    for images, labels in tqdm(testloader):
        images, labels = images.to(device), labels.to(device)
        outputs = model(images)
        _, predictions = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predictions == labels).sum().item()

print('Accuracy of the network on the 10000 test images: %d %%' % (
    100 * correct / total))

Output:
You're forgetting to call model.eval():
# ...

model.to(device)
model.eval() # <<<<<<<<<<<<<

total = 0
correct = 0
with torch.no_grad():
    for images, labels in tqdm(testloader):
        images, labels = images.to(device), labels.to(device)
        outputs = model(images)

# ...

As your model has BatchNorm layers, batch_size=1 is particularly degrading.
The pre-processing should also follow the one used for training. As you can see in the repository of the author of the model, you should normalize using the following statistics:
test_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=(0.4914, 0.4822, 0.4465), std=(0.2023, 0.1994, 0.2010))
])


Instruction: How to get output from intermediate encoder layers in PyTorch Transformer?

Input:
I have trained a fairly simple Transformer model with 6 TransformerEncoder layers:
class LitModel(pl.LightningModule):
    def __init__(self,
                 num_tokens: int,
                 dim_model: int = 96,
                 dim_h: int = 128,
                 n_head: int = 1,
                 dropout: float = 0.1,
                 activation: str = 'relu',
                 num_layers: int = 2,
                 lr: float=1e-3):
        """

        :param num_tokens:
        :param dim_model:
        :param dim_h:
        :param n_head:
        :param dropout:
        :param activation:
        :param num_layers:
        """
        super().__init__()
        self.lr = lr
        self.embed = torch.nn.Embedding(num_embeddings=num_tokens,
                                        embedding_dim=dim_model)
        encoder_layer = torch.nn.TransformerEncoderLayer(d_model=dim_model,
                                                         nhead=n_head,
                                                         dim_feedforward=dim_h,
                                                         dropout=dropout,
                                                         activation=activation,
                                                         batch_first=True)
        self.encoder = torch.nn.TransformerEncoder(encoder_layer=encoder_layer,
                                                   num_layers=num_layers)
        self.linear = torch.nn.Linear(in_features=dim_model, out_features=num_tokens)

    def forward(self, indices, mask):
        x = self.embed(indices)
        x = self.encoder(x, src_key_padding_mask=mask)
        return x

    def training_step(self, batch, batch_idx):
        x = batch['src']
        y = batch['label']
        mask = batch['mask']

        x = self.embed(x)
        x = self.encoder(x, src_key_padding_mask=mask)
        x = self.linear(x)

        loss = F.cross_entropy(input=x.transpose(1, 2),
                               target=y,
                               ignore_index=0)
        self.log('train_loss', loss)
        return loss

After training the model to predict [MASK] tokens (exactly like BERT), I would like to be able to extract the outputs from the lower layers, specifically, the second to last TransformerEncoderLayer, which may give a better vector encoding than the final layer (according to the original BERT paper). I'm not sure how to go about doing this.

Output:
Just in case it is not clear from the comments, you can do that by registering a forward hook:
activation = {}
def get_activation(name):
    def hook(model, input, output):
        activation[name] = output.detach()
    return hook

# instantiate the model
model = LitModel(...)

# register the forward hook
model.encoder.layers[-2].register_forward_hook(get_activation('encoder_penultimate_layer'))

# pass some data through the model
output = model(x)

# this is what you're looking for
activation['encoder_penultimate_layer']


Instruction: RTX3060 cannot run Pytorch Yolov4 with cuda11.4

Input:
Before I am using RTX2070 SUPER to run Pytorch Yolov4 and now my PC is changed to use RTX3060, ASUS KO GeForce RTX™ 3060 OC.
I have deleted the existing cuda11.2 and install again with cuda11.4 and Nvidia Driver 470.57.02
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.57.02    Driver Version: 470.57.02    CUDA Version: 11.4     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ...  On   | 00000000:07:00.0 Off |                  N/A |
|  0%   42C    P8    16W / 170W |    403MiB / 12053MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      1236      G   /usr/lib/xorg/Xorg                  9MiB |
|    0   N/A  N/A      1264      G   /usr/bin/gnome-shell                6MiB |
|    0   N/A  N/A      2124      C   python                            153MiB |
+-----------------------------------------------------------------------------+

However with the cuda11.4 and RTX3060, I cannot run Pytorch Yolov4 detection. When I run the detection, the detection will be stuck after loading weights, Loading weights from ./data/people.weights... Done!. In the meantime, nvidia-smi can show that a "python" (above PID 2124) is using the GPU memory and the used GPU memory of "python" will keep increasing.
Is cuda11.4 not support RTX3060 or Pytorch1.4 yet?
Environment:

ASUS KO GeForce RTX™ 3060 OC
Ubuntu 18.04.5 LTS

cuda 11.4
nvidia driver 470.57.02
conda 4.8.3
python 3.8.5
pytorch 1.4

Output:
Solved by reinstalling the pytorch in my Conda Env.
You may try reinstalling the Pytorch or create a new Conda Environment to do it again.


Instruction: How to train a deep learning model on a GPU server with laptop closed?

Input:
I'm about to train my own ASR model using ESPNet on a GPU server.
If my calculations are right, it's going to take about 4 consecutive days (using about 100G of audio data).
I'm mainly using VScode to remotely connect to the SSH server, and will run the shell file with the VScode terminal.
My question is that will I have to leave my laptop open for four days in order to train my model?
not sure if this is any useful info, but this is my nvcc --version:
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2019 NVIDIA Corporation
Built on Wed_Oct_23_19:24:38_PDT_2019
Cuda compilation tools, release 10.2, V10.2.89

and my nvidia-smi:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.33.01    Driver Version: 440.33.01    CUDA Version: 10.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Quadro RTX 6000     Off  | 00000000:00:06.0 Off |                    0 |
| N/A   32C    P0    41W / 250W |      0MiB / 22698MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

Once my data is all prepared, I'll execute the run.sh file.
Espnet github:
https://github.com/espnet/espnet
The model I'm using is located in espnet/egs2/zeroth_korean/asr1.
I'm fairly new to linux servers and models this heavy and large, so any type of feedback would be much appreciated.

Output:
Many Linux versions include the GNU Screen program, which - amongst other things - allow you to keep processes running after you've logged off.
Once connected, simply run the screen command:
[myhost ~]$ screen

Start your long running process inside this screen terminal.
You can now close the terminal. Power off, restart your computer, whatever.
When you want to check up on your process, just re-connect and run the following command to re-attach:
[myhost ~]$ screen -r

I hope this works for you.
screen has lots of other nice tricks. Just google "Linux Screen" for an abundance of articles on this.


Instruction: Custom Images with GAN

Input:
I downloaded this image set https://www.kaggle.com/jessicali9530/stanford-dogs-dataset
and extracted those images folder in to my data folder
So now it's like this

Below is my code
from __future__ import print_function
import torch.nn as nn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import os

batchSize = 64  # We set the size of the batch.
imageSize = 64  # We set the size of the generated images (64x64).

# Creating the transformations
transform = transforms.Compose([transforms.Scale(imageSize), transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5,
                                                                       0.5)), ])  # We create a list of transformations (scaling, tensor conversion, normalization) to apply to the input images.

# Loading the dataset
dataset = dset.ImageFolder(root='./data', transform=transform)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batchSize, shuffle=True,
                                         num_workers=2)  # We use dataLoader to get the images of the training set batch by batch.

# Defining the weights_init function that takes as input a neural network m and that will initialize all its weights.
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

input_vector = 100

# Create results directory
def create_dir(name):
    if not os.path.exists(name):
        os.makedirs(name)

# Defining the Generator
class G(nn.Module):
    feature_maps = 512
    kernel_size = 4
    stride = 2
    padding = 1
    bias = False

    def __init__(self):
        super(G, self).__init__()
        self.main = nn.Sequential(
            nn.ConvTranspose2d(input_vector, self.feature_maps, self.kernel_size, 1, 0, bias=self.bias),
            nn.BatchNorm2d(self.feature_maps), nn.ReLU(True),
            nn.ConvTranspose2d(self.feature_maps, int(self.feature_maps // 2), self.kernel_size, self.stride, self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int(self.feature_maps // 2)), nn.ReLU(True),
            nn.ConvTranspose2d(int(self.feature_maps // 2), int((self.feature_maps // 2) // 2), self.kernel_size, self.stride,
                               self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int((self.feature_maps // 2) // 2)), nn.ReLU(True),
            nn.ConvTranspose2d((int((self.feature_maps // 2) // 2)), int(((self.feature_maps // 2) // 2) // 2), self.kernel_size,
                               self.stride, self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int((self.feature_maps // 2) // 2) // 2), nn.ReLU(True),
            nn.ConvTranspose2d(int(((self.feature_maps // 2) // 2) // 2), 3, self.kernel_size, self.stride, self.padding,
                               bias=self.bias),
            nn.Tanh()
        )

    def forward(self, input):
        output = self.main(input)
        return output

# Creating the generator
netG = G()
netG.apply(weights_init)

class D(nn.Module):
    feature_maps = 64
    kernel_size = 4
    stride = 2
    padding = 1
    bias = False
    inplace = True

    def __init__(self):
        super(D, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(3, self.feature_maps, self.kernel_size, self.stride, self.padding, bias=self.bias),
            nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps, self.feature_maps * 2, self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * 2), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * 2, self.feature_maps * (2 * 2), self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2), self.feature_maps * (2 * 2 * 2), self.kernel_size, self.stride,
                      self.padding, bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2 * 2), 1, self.kernel_size, 1, 0, bias=self.bias),
            nn.Sigmoid()
        )

    def forward(self, input):
        output = self.main(input)
        return output.view(-1)

# Creating the discriminator
netD = D()
netD.apply(weights_init)

# Training the DCGANs
criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters(), lr=0.002, betas=(0.5, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=0.002, betas=(0.5, 0.999))

nb_epochs = 25

for epoch in range(25):

    for i, data in enumerate(dataloader, 0):

        # 1st Step: Updating the weights of the neural network of the discriminator

        netD.zero_grad()

        # Training the discriminator with a real image of the dataset
        real, _ = data
        input = Variable(real)
        target = Variable(torch.ones(input.size()[0]))
        output = netD(input)
        errD_real = criterion(output, target)

        # Training the discriminator with a fake image generated by the generator
        noise = Variable(torch.randn(input.size()[0], 100, 1, 1))
        fake = netG(noise)
        target = Variable(torch.zeros(input.size()[0]))
        output = netD(fake.detach())
        errD_fake = criterion(output, target)

        # Backpropagating the total error
        errD = errD_real + errD_fake
        errD.backward()
        optimizerD.step()

        # 2nd Step: Updating the weights of the neural network of the generator

        netG.zero_grad()
        target = Variable(torch.ones(input.size()[0]))
        output = netD(fake)
        errG = criterion(output, target)
        errG.backward()
        optimizerG.step()

        # 3rd Step: Printing the losses and saving the real images and the generated images of the minibatch every 100 steps

        print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f' % (epoch, 25, i, len(dataloader), errD.data[0], errG.data[0]))
        if i % 100 == 0:
            vutils.save_image(real, '%s/real_samples.png' % "./results", normalize=True)
            fake = netG(noise)
            vutils.save_image(fake.data, '%s/fake_samples_epoch_%03d.png' % ("./results", epoch), normalize=True)

But when I run that it gives me this error
RuntimeError: stack expects each tensor to be equal size, but got [3, 64, 85] at entry 0 and [3, 64, 80] at entry 1

That means the issue is with the for loop.
How can I change that for loop to match my dataset ?

Output:
try using transforms.Resize instead of transforms.Scale(imageSize), as Resize behaves differently on input images with a different height and width. Pass the size argument as a tuple:
....
transforms.Resize((imageSize, imageSize))
....


Instruction: Cannot import modules from torch

Input:
I have installed torch 1.4.0. It has worked with all modules for 4 months well. But yesterday I ran into a problem in Jupyter Notebook such as:
import torch
import torch.nn as nn
import torch.optim as optim

And it throws "cannot find module torch.optim". But it could import torch and torch.nn! I can't understand why. I didn't change venv or install other packages. Same problem with torch.nn.Functional and torch.onnx. Does anybody know how to solve this problem?

Output: So the problem solved with installing torch 1.6.0. I have no idea why this happened with torch 1.4.0. Anyway now it works.


Instruction: Learning Object Detection Detected result showed in discolouration

Input:
Breif Description
Recently begin to learning Object Detection, Just starting off with PyTorch, YOLOv5. So I thought why not build a small side project to learn? Using it to train to detect Pikachu.
The Problem
I've successfully trained the model with Pikachu and then uses trained weights with myself written Python script/code to detect the Pikachu using test images, now, here's the problem, Pikachus can successfully detected but all the result showed in blue discolouration, what supposed to be yellow, all turned into blue and blue into yellow.
Fig-1 Result images showed in blue discolouration(few example outputs)
Additional Imformation
I've pushed this project to the GitHub, feel free to download it or pull it to debugging.
GitHub repository where it contains all the files
Any solution/suggestion would be helpful, Thanks.
The Code

"""Object detection using YOLOv5

Pokemon Pikachu detecting

"""

# import os, sys to append YOLOv5 folder path
import os, sys

# import object detection needed modules and libraries
# pillow
from PIL import Image, ImageDraw, ImageFont
import numpy as np
import torch # PyTorch

# YOLOv5 folder path and related folder path settings
cwd = os.getcwd()
root_dir = (cwd + "/yolov5_stable")
sys.path.append(root_dir)

# import methods, functions from YOLOv5
from models.experimental import attempt_load
from utils.datasets import LoadImages
from utils.general import non_max_suppression, scale_coords
from utils.plots import colors

# define a function to show detected pikachu
def show_pikachu(img, det):
    labels = ["pikachu"]
    img = Image.fromarray(img)
    draw = ImageDraw.Draw(img)
    font_size = max(round(max(img.size)/40), 12)
    font = ImageFont.truetype(cwd + "/yolov5_stable/fonts/times.ttf")

    for info in det:
        color = colors(1)
        target, prob = int(info[5].cpu().numpy()), np.round(info[4].cpu().numpy(), 2)
        x_min, y_min, x_max, y_max = info[0], info[1], info[2], info[3]
        draw.rectangle([x_min, y_min, x_max, y_max], width = 3, outline = color)
        draw.text((x_min, y_min), labels[target] + ':' + str(prob), fill = color, font = font)

    # Bug unresolved, pikachu shown in blue discolouration

    return img

if __name__ == "__main__":
    device = "cuda:0" if torch.cuda.is_available() else "cpu"
    print("GPU State: ", device)

    data_path = (cwd + "/test_data/")
    weight_path = (cwd + "/yolov5_stable/weights/best_v1.pt")
    dataset = LoadImages(data_path)
    model = attempt_load(weight_path, map_location = device)
    model.to(device)

    for path, img, im0s, _ in dataset:
        img = torch.from_numpy(img).to(device)
        img = img.float() # uint8 to fp16/32
        img /= 255.0 # 0-255 to 0.0-1.0
        if img.ndimension() == 3:
            img = img.unsqueeze(0)

        pred = model(img)[0]
        pred = non_max_suppression(pred, 0.25, 0.45)
        for i, det in enumerate(pred):
            im0 = im0s.copy()
            det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape).round()
            result = show_pikachu(im0, det)
            result.show()

Output:
The problem is that Image.fromarray expects image in RGB and you're providing them in BGR. You just need to change that. There are multiple places you could do that, for instance:
Image.fromarray(img[...,::-1])  # assuming `img` is channel-last

An evidence is that the red parts of the mouse (red is RGB(255, 0, 0)) are being shown in blue (which is RGB(0, 0, 255)). FYI, yellow is RGB(255, 255, 0) and cyan is RGB(0, 255, 255), which you can also see in your case.


Instruction: Load a saved model on Pytorch AttributeError: 'dict' object has no attribute 'seek'

Input:
I trained a model, and I saved the best one, with:
# Store params at the best validation accuracy
if save_param and accuracy_star > best_accuracy:
    torch.save({'vgg_a':nets[0].state_dict(),'classifier':nets[3].state_dict()}, f"{model_name}_best_test.pth")

So, at the end of the training I downloaded this "federated_mnist_best_test.pth" on my local computer.
Now, I created a new notebook on google colab, and I tried din this way to upload the model:
from google.colab import files
uploaded = files.upload()

Inside uploaded I put my .pth file, and then:
state_dict = torch.load(uploaded)

gives me this error:
AttributeError: 'dict' object has no attribute 'seek'. You can only torch.load from a file that is seekable. Please pre-load the data into a buffer like io.BytesIO and try to load from it instead.

Output:
After having called uploaded = files.upload() (usage examples can be found on this notebook) and interacted with the file explorer, the file will be uploaded to google Colaboratory's temporary file system.
You can find it by looking at the side panel:

At this point, the file has been uploaded to the file system but hasn't yet been loaded on the notebook. You need to load the file using its name (e.g. if the name is best_test.pth):
state_dict = torch.load('best_test.pth')


Instruction: Extend a tensor of sequences so that every sequence is prepended

Input:
I am having a tensor of 3 sequences where every sequence has length 2 and consists of vectors of size 2:
import torch
t = torch.Tensor([[[11,12],[21,22]], [[31,32],[41,42]], [[51,52],[61,62]]])
>>> t
tensor([[[11., 12.],
         [21., 22.]],

        [[31., 32.],
         [41., 42.]],

        [[51., 52.],
         [61., 62.]]])

So t has the structure t[batch, sequencePos, dataPos]. How can I extend every sequence so that it is prepended by a new element [01, 02] (sequence have then length 3) so that I get:
tensor([[[01., 02.],
         [11., 12.],
         [21., 22.]],

        [[01., 02.],
         [31., 32.]
         [41., 42.]],

        [[01., 02.],
         [51., 52.],
         [61., 62.]]])

Output:
You are looking to concatenate two tensors on axis=1, the first one being t:
 >>> t
tensor([[[11., 12.],
         [21., 22.]],

        [[31., 32.],
         [41., 42.]],

        [[51., 52.],
         [61., 62.]]])

The second one is an arrangement:
>>> arr = torch.arange(t.size(-1))
tensor([0, 1])

However, we first need to broadcast it to the correct shape using torch.reshape and torch.repeat:
>>> arr = arr.reshape(1, 1, -1).repeat(len(t), 1, 1)
tensor([[[0, 1]],

        [[0, 1]],

        [[0, 1]]])

At this point, arr.shape is torch.Size([3, 1, 4]).
We are set for concatenating arr and t together, either with torch.cat:
>>> torch.cat((arr, t), dim=1)

or more elegantly with torch.hstack:
>>> torch.hstack((arr, t))
tensor([[[ 0.,  1.],
         [11., 12.],
         [21., 22.]],

        [[ 0.,  1.],
         [31., 32.],
         [41., 42.]],

        [[ 0.,  1.],
         [51., 52.],
         [61., 62.]]])

Notice how this implementation will work with any 3-dimensional input. In the following example t has three columns instead of two:
>>> t
tensor([[[11., 12., 13.],
         [21., 22., 23.]],

        [[31., 32., 33.],
         [41., 42., 43.]],

        [[51., 52., 53.],
         [61., 62., 63.]]])

>>> arr = torch.arange(t.size(-1)).reshape(1, 1, -1).repeat(len(t), 1, 1)
>>> torch.cat((arr, t), dim=1)
tensor([[[ 0.,  1.,  2.],
         [11., 12., 13.],
         [21., 22., 23.]],

        [[ 0.,  1.,  2.],
         [31., 32., 33.],
         [41., 42., 43.]],

        [[ 0.,  1.,  2.],
         [51., 52., 53.],
         [61., 62., 63.]]])

This can even be generalized to n-dimensional tensors. You just have to take care of the reshape and repeat calls where the number of arguments depends on the number of dimensions.
>>> ones = (1,)*(t.ndim-1)
>>> arr = torch.arange(t.size(-1)).reshape(*ones, -1).repeat(len(t), *ones)
>>> torch.cat((arr, t), dim=-2)


Instruction: python module not found after executing shell script even though the module is installed

Input:
pip3 list
Package             Version
------------------- ------------
apipkg              1.5
apparmor            3.0.3
appdirs             1.4.4
asn1crypto          1.4.0
brotlipy            0.7.0
certifi             2021.5.30
cffi                1.14.6
chardet             4.0.0
cmdln               2.0.0
configobj           5.0.6
createrepo-c        0.17.3
cryptography        3.3.2
cssselect           1.1.0
cupshelpers         1.0
cycler              0.10.0
decorator           5.0.9
idna                3.2
iniconfig           0.0.0
isc                 2.0
joblib              1.0.1
kiwisolver          1.3.1
LibAppArmor         3.0.3
lxml                4.6.3
matplotlib          3.4.3
mysqlclient         2.0.3
nftables            0.1
notify2             0.3.1
numpy               1.21.1
opi                 2.1.1
ordered-set         3.1.1
packaging           20.9
pandas              1.3.1
Pillow              8.3.1
pip                 20.2.4
ply                 3.11
psutil              5.8.0
py                  1.10.0
pyasn1              0.4.8
pycairo             1.20.1
pycparser           2.20
pycups              2.0.1
pycurl              7.43.0.6
PyGObject           3.40.1
pyOpenSSL           20.0.1
pyparsing           2.4.7
pysmbc              1.0.23
PySocks             1.7.1
python-dateutil     2.8.2
python-linux-procfs 0.6
pytz                2021.1
pyudev              0.22.0
requests            2.25.1
rpm                 4.16.1.3
scikit-learn        0.24.2
scipy               1.7.1
setuptools          57.4.0
six                 1.16.0
sklearn             0.0
slip                0.6.5
slip.dbus           0.6.5
termcolor           1.1.0
threadpoolctl       2.2.0
torch               1.9.0+cu111
torchaudio          0.9.0
torchvision         0.10.0+cu111
tqdm                4.62.1
typing-extensions   3.10.0.0
urllib3             1.26.6

Above shows my installed modules, but when i go into the project folder, and run the shell script, I get:
    Traceback (most recent call last):
  File "main.py", line 3, in <module>
    import torch
ImportError: No module named torch

Even though in the list above it clearly shows that torch is installed.
Please help. My $PATH is /usr/bin/python:/home/anthony/bin:/usr/local/bin:/usr/bin:/bin:/snap/bin
and my printenv PYTHONPATH is :/usr/bin/python
Let me know if you need any other print outs, I've tried everything, and nothing seems to be working. I am working mainly in pycharm.

Output:
It is very likely that pip3 is pointing to a different python instance.
Imagine you had python, python3, python3.6 and python3.8 all installed on your system. Which one would pip3 install packages for? (who knows?)
It is almost always safer to do python3.8 -m pip list/install since you can be sure that python3.8 somefile.py will be using the same files you just saw. (even better, do python3.8 -m venv /path/to/some/virtualenv and then make sure that is activated, then you can be sure pip points to the same python)


Instruction: Where is the "negative" slope in a LeakyReLU?

Input:
What does the negative slope in a LeakyReLU function refer to?
The term "negative slope" is used in the documentation of both TensorFlow and Pytorch, but it does not seem to point to reality.
The slope of a LeakyReLU function for both positive and negative inputs is generally non-negative.
The Pytorch and TensorFlow docs provide examples of setting the negative slope, and they both use a positive value. TensorFlow explicitly enforces non-negative values. (See below)
Are they just wrong or am I missing something?
Pytorch docs:
CLASStorch.nn.LeakyReLU(negative_slope=0.01, inplace=False)

Tensorflow LeakyReLU:

Args  alpha   Float >= 0. Negative slope coefficient. Default to 0.3.

Output:
negative_slope in this context means the negative half of the Leaky ReLU's slope. It is not describing a slope which is necessarily negative.
When naming kwargs it's normal to use concise terms, and here "negative slope" and "positive slope" refer to the slopes of the linear splines spanning the negative [-∞,0] and positive (0,∞] halves of the Leaky ReLU's domain.


Instruction: UnicodeDecodeError: ‘utf-8’ codec can’t decode byte 0xbe in position 2: invalid start byte

Input:
Do you know how I can fix this problem in PyTorch 1.9?
File "main.py", line 138, in main
    checkpoint = torch.load(args.resume)
  File "/scratch3/venv/fashcomp/lib/python3.8/site-packages/torch/serialization.py", line 608, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
  File "/scratch3/venv/fashcomp/lib/python3.8/site-packages/torch/serialization.py", line 787, in _legacy_load
    result = unpickler.load()
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbe in position 2: invalid start byte

I have:
$ pip freeze
h5py==3.3.0
joblib==1.0.1
numpy==1.21.2
Pillow==8.3.1
scikit-learn==0.24.2
scipy==1.7.1
sklearn==0.0
threadpoolctl==2.2.0
torch==1.9.0
torchaudio==0.9.0
torchvision==0.10.0
typing-extensions==3.10.0.0

Output:
Can you try something like this:
>>> torch.load('model.pt', encoding='ascii')  # or latin1, or other encoding

By default, we decode byte strings as utf-8. This is to avoid a common error case UnicodeDecodeError: 'ascii' codec can't decode byte 0x... when loading files saved by Python 2 in Python 3. If this default is incorrect, you may use an extra encoding keyword argument to specify how these objects should be loaded, e.g., encoding='latin1' decodes them to strings using latin1 encoding, and encoding='bytes' keeps them as byte arrays which can be decoded later with byte_array.decode(...).

Read this, the 2nd note.


Instruction: The expanded size of the tensor (64) must match the existing size (66) at non-singleton dimension 2

Input:
When I run the following (unchanged) code in stable PyTorch 1.9, I get the following error. How can I diagnose and fix it?
(fashcomp) [jalal@goku fashion-compatibility]$ python main.py --test --l2_embed --resume runs/nondisjoint_l2norm/model_best.pth.tar --datadir ../../../data/fashion/
/scratch3/venv/fashcomp/lib/python3.8/site-packages/torchvision/transforms/transforms.py:310: UserWarning: The use of the transforms.Scale transform is deprecated, please use transforms.Resize instead.
  warnings.warn("The use of the transforms.Scale transform is deprecated, " +
=> loading checkpoint 'runs/nondisjoint_l2norm/model_best.pth.tar'
=> loaded checkpoint 'runs/nondisjoint_l2norm/model_best.pth.tar' (epoch 5)
/scratch3/venv/fashcomp/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)
Traceback (most recent call last):
  File "main.py", line 312, in <module>
    main()
  File "main.py", line 149, in main
    test_acc = test(test_loader, tnet)
  File "main.py", line 244, in test
    embeddings.append(tnet.embeddingnet(images).data)
  File "/scratch3/venv/fashcomp/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "/scratch3/research/code/fashion/fashion-compatibility/type_specific_network.py", line 119, in forward
    masked_embedding = masked_embedding / norm.expand_as(masked_embedding)
RuntimeError: The expanded size of the tensor (64) must match the existing size (66) at non-singleton dimension 2.  Target sizes: [256, 66, 64].  Tensor sizes: [256, 66]

Related issue: https://github.com/mvasil/fashion-compatibility/pull/13

Output:
The problem is that norm is missing that extra dimension: masked_embedding is [256, 66, 64] and norm is [256, 66].  You can fix that by adding this extra dimension ([256, 66, 1]) to norm this way:
masked_embedding = masked_embedding / norm.unsqueeze(-1).expand_as(masked_embedding)

OR, by modifying the call (this one) that generated norm:
norm = torch.norm(masked_embedding, p=2, dim=2, keepdim=True) + 1e-10


Instruction: How does calculation in a GRU layer take place

Input:
So I want to understand exactly how the outputs and hidden state of a GRU cell are calculated.
I obtained the pre-trained model from here and the GRU layer has been defined as nn.GRU(96, 96, bias=True).
I looked at the the PyTorch Documentation and confirmed the dimensions of the weights and bias as:

weight_ih_l0: (288, 96)
weight_hh_l0: (288, 96)
bias_ih_l0: (288)
bias_hh_l0: (288)

My input size and output size are (1000, 8, 96). I understand that there are 1000 tensors, each of size (8, 96). The hidden state is (1, 8, 96), which is one tensor of size (8, 96).
I have also printed the variable batch_first and found it to be False. This means that:

Sequence length: L=1000
Batch size: B=8
Input size: Hin=96

Now going by the equations from the documentation, for the reset gate, I need to multiply the weight by the input x. But my weights are 2-dimensions and my input has three dimensions.
Here is what I've tried, I took the first (8, 96) matrix from my input and multiplied it with the transpose of my weight matrix:
Input (8, 96) x Weight (96, 288) = (8, 288)

Then I add the bias by replicating the (288) eight times to give (8, 288). This would give the size of r(t) as (8, 288). Similarly, z(t) would also be (8, 288).
This r(t) is used in n(t), since Hadamard product is used, both the matrices being multiplied have to be the same size that is (8, 288). This implies that n(t) is also (8, 288).
Finally, h(t) is the Hadamard produce and matrix addition, which would give the size of h(t) as (8, 288) which is wrong.
Where am I going wrong in this process?

Output:
TLDR; This confusion comes from the fact that the weights of the layer are the concatenation of input_hidden and hidden-hidden respectively.

- nn.GRU layer weight/bias layout
You can take a closer look at what's inside the GRU layer implementation torch.nn.GRU by peaking through the weights and biases.
>>> gru = nn.GRU(input_size=96, hidden_size=96, num_layers=1)

First the parameters of the GRU layer:
>>> gru._all_weights
[['weight_ih_l0', 'weight_hh_l0', 'bias_ih_l0', 'bias_hh_l0']]

You can look at gru.state_dict() to get the dictionary of weights of the layer.
We have two weights and two biases, _ih stands for 'input-hidden' and _hh stands for 'hidden-hidden'.
For more efficient computation the parameters have been concatenated together, as the documentation page clearly explains (| means concatenation). In this particular example num_layers=1 and k=0:

~GRU.weight_ih_l[k] – the learnable input-hidden weights of the layer (W_ir | W_iz | W_in), of shape (3*hidden_size, input_size).

~GRU.weight_hh_l[k] – the learnable hidden-hidden weights of the layer (W_hr | W_hz | W_hn), of shape (3*hidden_size, hidden_size).

~GRU.bias_ih_l[k] – the learnable input-hidden bias of the layer (b_ir | b_iz | b_in), of shape (3*hidden_size).

~GRU.bias_hh_l[k] – the learnable hidden-hidden bias of the (b_hr | b_hz | b_hn).

For further inspection we can get those split up with the following code:
>>> W_ih, W_hh, b_ih, b_hh = gru._flat_weights
>>> W_ir, W_iz, W_in = W_ih.split(H_in)
>>> W_hr, W_hz, W_hn = W_hh.split(H_in)
>>> b_ir, b_iz, b_in = b_ih.split(H_in)
>>> b_hr, b_hz, b_hn = b_hh.split(H_in)

Now we have the 12 tensor parameters sorted out.

- Expressions
The four expressions for a GRU layer: r_t, z_t, n_t, and h_t, are computed at each timestep.
The first operation is r_t = σ(W_ir@x_t + b_ir + W_hr@h + b_hr). I used the @ sign to designate the matrix multiplication operator (__matmul__). Remember W_ir is shaped (H_in=input_size, hidden_size) while x_t contains the element at step t from the x sequence. Tensor x_t = x[t] is shaped as (N=batch_size, H_in=input_size). At this point, it's simply a matrix multiplication between the input x[t] and the weight matrix. The resulting tensor r is shaped (N, hidden_size=H_in):
>>> (x[t]@W_ir.T).shape
(8, 96)

The same is true for all other weight multiplication operations performed. As a result, you end up with an output tensor shaped (N, H_out=hidden_size).
In the following expressions h is the tensor containing the hidden state of the previous step for each element in the batch, i.e. shaped (N, hidden_size=H_out), since num_layers=1, i.e. there's a single hidden layer.
>>> r_t = torch.sigmoid(x[t]@W_ir.T + b_ir + h@W_hr.T + b_hr)
>>> r_t.shape
(8, 96)

>>> z_t = torch.sigmoid(x[t]@W_iz.T + b_iz + h@W_hz.T + b_hz)
>>> z_t.shape
(8, 96)

The output of the layer is the concatenation of the computed h tensors at
consecutive timesteps t (between 0 and L-1).

- Demonstration
Here is a minimal example of an nn.GRU inference manually computed:

Parameters
Description
Values

H_in
feature size
3

H_out
hidden size
2

L
sequence length
3

N
batch size
1

k
number of layers
1

Setup:
gru = nn.GRU(input_size=H_in, hidden_size=H_out, num_layers=k)
W_ih, W_hh, b_ih, b_hh = gru._flat_weights
W_ir, W_iz, W_in = W_ih.split(H_out)
W_hr, W_hz, W_hn = W_hh.split(H_out)
b_ir, b_iz, b_in = b_ih.split(H_out)
b_hr, b_hz, b_hn = b_hh.split(H_out)

Random input:
x = torch.rand(L, N, H_in)

Inference loop:
output = []
h = torch.zeros(1, N, H_out)
for t in range(L):
   r = torch.sigmoid(x[t]@W_ir.T + b_ir + h@W_hr.T + b_hr)
   z = torch.sigmoid(x[t]@W_iz.T + b_iz + h@W_hz.T + b_hz)
   n = torch.tanh(x[t]@W_in.T + b_in + r*(h@W_hn.T + b_hn))
   h = (1-z)*n + z*h
   output.append(h)

The final output is given by the stacking the tensors h at consecutive timesteps:
>>> torch.vstack(output)
tensor([[[0.1086, 0.0362]],

        [[0.2150, 0.0108]],

        [[0.3020, 0.0352]]], grad_fn=<CatBackward>)

In this case the output shape is (L, N, H_out), i.e. (3, 1, 2).
Which you can compare with output, _ = gru(x).


Instruction: How to make a custom DataLoader return values and labels seperately?

Input:
I want to iterate over a custom DataLoader using batches with matching values and labels. Modification of PandasDataset described below is needed and since I copied it from online I do not have a great grasp of how it works
import torch
import pandas as pd
from torch.utils.data import Dataset
from torch.utils.data import DataLoader

class PandasDataset(Dataset):
    def __init__(self, dataframe):
        self.dataframe = dataframe

    def __len__(self):
        return len(self.dataframe)

    def __getitem__(self, index):
        return self.dataframe.iloc[index]

d = {'values': [1, 2], 'values2': [3, 4],'labels': [5, 6]}
df = pd.DataFrame(data=d)
dataset = PandasDataset(df)
loader = DataLoader(torch.tensor(dataset), batch_size=1, shuffle=False)

for batch_index, (values, label) in enumerate(loader):
    print(values)
    print(label)

Output:
You can change __getitem__ to something like this:
    def __getitem__(self, index):
        data = self.dataframe.iloc[index].to_numpy()
        return data[:-1], data[-1]

Then, you don't need to wrap your dataset with torch.tensor:
loader = DataLoader(dataset, batch_size=1, shuffle=False)

and it'll return:
next(iter(loader))
# >>> [tensor([[1, 3]]), tensor([5])]


Instruction: How to do Normalization in CNN?

Input:
I am new to CNN, and I am learning it with Food Classification. Here is my code. In the DATASET part, I change the train dataset and validation dataset from numpy to tensor. At that point, the shape of tensor is ([9866, 128, 128, 3]). Since the channel 3 need to be in the first index, so I use "transpose" method to change the index. Then, I used the "Data.TensorDataset" to put training data and training label together, and the reason to use "Data.DataLoader" is that I need the batch size to speed up.
import os
import numpy as np
import pandas as pd
import cv2
import torch
import torch.nn as nn
from torch.nn import functional as F
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch import optim
import pandas as pd
from torch.utils.data import DataLoader, Dataset
import torch.utils.data as Data

'''Initialize Params'''
epochs = 3
learning_rate = 0.0001
momentum = 0.5
batch_size = 128

'''Load Data'''
def readFile(path,label):
    image_dir = sorted(os.listdir(path))
    # x stores photos
    x = np.zeros((len(image_dir),128,128,3),dtype=np.uint8)
    # y stores labels
    y = np.zeros((len(image_dir)), dtype=np.uint8)
    for i, file in enumerate(image_dir):
        img = cv2.imread(os.path.join(path, file))
        x[i, :, :] = cv2.resize(img,(128, 128))
        if label:
            y[i] = int(file.split("_")[0])
    if label:
        return x,y
    else:
        return x

train_x, train_y = readFile('./food/training',True)
val_x, val_y = readFile('./food/validation',True)
test_x = readFile('./food/testing',False)
# print("Reading data: ")
# print("Size of training data = {}".format(len(train_x)))
# print("Size of validation data = {}".format(len(val_x)))
# print("Size of Testing data = {}".format(len(test_x)))

'''DataSet'''
train_x = torch.tensor(train_x)
# print(train_x.shape)
train_x = train_x.transpose(1,3).float()
train_y = torch.tensor(train_y)
val_x = torch.tensor(val_x)
val_x = val_x.transpose(1, 3).float()
val_y = torch.tensor(val_y)

train_dataset = Data.TensorDataset(train_x,train_y)
val_dataset = Data.TensorDataset(val_x,val_y)

train_loader = Data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
val_loader = Data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=True)

I got the 68% accuracy of training set and I would like to improve it. I searched on net and found that maybe I should add the normalization. But I only found the way like this
transform = transforms.Compose([
    transforms.ToTensor(), # range [0, 255] -> [0.0,1.0]
    ]
)

and I confuse about how to put it with "Data.DataLoader".
And I knew that there is a other way to change the training data from numpy to dataloader like this, and here is the link
train_transform = transforms.Compose([
    transforms.ToPILImage(),
    transforms.RandomHorizontalFlip(),
    transforms.RandomRotation(15),
    transforms.ToTensor(),
])

test_transform = transforms.Compose([
    transforms.ToPILImage(),
    transforms.ToTensor(),
])

class ImgDataset(Dataset):
    def __init__(self, x, y=None, transform=None):
        self.x = x
        self.y = y
        if y is not None:
            self.y = torch.LongTensor(y)
        self.transform = transform
    def __len__(self):
        return len(self.x)
    def __getitem__(self, index):
        X = self.x[index]
        if self.transform is not None:
            X = self.transform(X)
        if self.y is not None:
            Y = self.y[index]
            return X, Y
        else:
            return X

train_set = ImgDataset(train_x, train_y, train_transform)
val_set = ImgDataset(val_x, val_y, test_transform)
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False)

My way might be silly but I would like to try it, and any help would be appreciated. I hope I explain it well, and thanks advance.
Here is the complete code if needed:
import os
import numpy as np
import pandas as pd
import cv2
import torch
import torch.nn as nn
from torch.nn import functional as F
import torchvision.transforms as transforms
from torch.autograd import Variable
from torch import optim
import pandas as pd
from torch.utils.data import DataLoader, Dataset
import torch.utils.data as Data

'''Initialize Params'''
epochs = 3
learning_rate = 0.0001
momentum = 0.5
batch_size = 128

transform = transforms.Compose([
    transforms.ToTensor(), # range [0, 255] -> [0.0,1.0]
    ]
)

'''Load Data'''
def readFile(path,label):
    image_dir = sorted(os.listdir(path))
    # x stores photos
    x = np.zeros((len(image_dir),128,128,3),dtype=np.uint8)
    # y stores labels
    y = np.zeros((len(image_dir)), dtype=np.uint8)
    for i, file in enumerate(image_dir):
        img = cv2.imread(os.path.join(path, file))
        x[i, :, :] = cv2.resize(img,(128, 128))
        if label:
            y[i] = int(file.split("_")[0])
    if label:
        return x,y
    else:
        return x

train_x, train_y = readFile('./food/training',True)
val_x, val_y = readFile('./food/validation',True)
test_x = readFile('./food/testing',False)
# print("Reading data: ")
# print("Size of training data = {}".format(len(train_x)))
# print("Size of validation data = {}".format(len(val_x)))
# print("Size of Testing data = {}".format(len(test_x)))

'''DataSet'''
train_x = torch.tensor(train_x)
# print(train_x.shape)
train_x = train_x.transpose(1,3).float()
train_y = torch.tensor(train_y)
val_x = torch.tensor(val_x)
val_x = val_x.transpose(1, 3).float()
val_y = torch.tensor(val_y)

train_dataset = Data.TensorDataset(train_x,train_y)
val_dataset = Data.TensorDataset(val_x,val_y)

train_loader = Data.DataLoader(dataset=train_dataset,batch_size=batch_size,shuffle=True)
val_loader = Data.DataLoader(dataset=val_dataset,batch_size=batch_size,shuffle=True)

'''Create Model'''
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # nn.Conv2d(input_channel, output_channel, kernel, stride)
        self.conv1 = nn.Conv2d(3,64,5,1,1)
        nn.BatchNorm2d(64)
        self.conv2 = nn.Conv2d(64,128,5,1,1)
        nn.BatchNorm2d(128)
        self.conv3 = nn.Conv2d(128,256,5,1,1)
        nn.BatchNorm2d(256)
        self.conv4 = nn.Conv2d(256,256,5,1,1)
        nn.BatchNorm2d(256)
        self.conv4_drop = nn.Dropout2d()
        self.fc1 = nn.Linear(6*6*256, 1024)
        # self.fc1 = nn.Linear(512*4*4, 1024)
        self.fc2 = nn.Linear(1024, 512)
        self.fc3 = nn.Linear(512, 256)
        self.fc4 = nn.Linear(256, 11)

    def forward(self, x):
        # maxpooling 1
        x = self.conv1(x)
        x = F.relu(x) # 124*124*64
        x = F.max_pool2d(x, 2) # 62*62*20

        # maxpooling 2
        x = self.conv2(x)
        x = F.relu(x) # 58*58*128
        x = F.max_pool2d(x, 2) # 29*29*40

        # maxpooling 3
        x = self.conv3(x)
        x = F.relu(x) # 25*25*256
        x = F.max_pool2d(x, 2) # 12*12*100

        # maxpooling 4
        x = self.conv4(x)
        x = F.relu(x) # 8*8*256
        x = F.max_pool2d(x, 2) # 4*4*256

        x = x.view(-1,6*6*256)
        # print(x.shape)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = self.fc4(x)
        # return F.log_softmax(x)
        return F.log_softmax(x)

'''Initialize the network'''
net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=learning_rate)

'''Train Model'''
if __name__ == '__main__':
    # print(train_x.shape)  torch.Size([9866, 128, 128, 3])
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    net.to(device)
    for epoch in range(epochs):
        for i,data in enumerate(train_loader):
            x,y = data
            # print(x)
            # print('--------------------')
            sum_loss = 0.0
            optimizer.zero_grad()
            x,y = Variable(x).cuda(), Variable(y).cuda()
            outputs = net(x)
            loss = criterion(outputs, y.long())
            loss.backward()
            optimizer.step()
            # print(loss.item())
            # sum_loss += loss.item()
            # if i % 500 == 99:
            #     print('[%d,%d] loss:%.03f' %
            #             (epoch + 1, i + 1, sum_loss / 500))
            #     sum_loss = 0.0

        '''Test Model with Validation'''
        net.eval()
        correct = 0
        total = 0
        for val_data in val_loader:
            x,y = val_data
            x, y = Variable(x).cuda(), Variable(y).cuda()
            output_test = net(x)
            _, predicted = torch.max(output_test, 1)
            total += y.size(0)
            correct += (predicted == y).sum()
        print("correct1: ", correct)
        print("Test acc: {0}".format(correct.item() /len(val_loader)))

Output:
The data normalization performed on the model's inputs (whether it'd be your training, validation, or testing set) depends on your training data statistics. More specifically, it corresponds to the channel-wise mean and standard deviation of the images used in the training set.

torchvision.transforms.Normalize
You can do so with torchvision's transforms.Normalize. It is a very easy function, yet a lot of people seem to have trouble understanding it - or fail to read its documentation...

torchvision.transforms.Normalize(mean, std, inplace=False)
Normalize a tensor image with mean and standard deviation. This transform does not support PIL Image. Given mean: (mean[1], ..., mean[n]) and std: (std[1], ..., std[n]) for n channels, this transform will normalize each channel of the input torch.*Tensor i.e., output[channel] = (input[channel] - mean[channel]) / std[channel]

You can argue the naming for both arguments 'mean' and 'std' is inaccurate is rather misleading indeed. This can explain why it's so misused (today again).
It is nothing more than a channel-wise shift-scale operator.

Misusage of the transform
Very frequently I see this type of normalization setup.
T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))

To clarify, the above won't do the following two things:

it won't normalize your input to [0, 1].

it won't standardize your input (i.e. make mean=0, std=1)

The described transform will map values from [0, 1] to [-1, 1] (since (x - .5) / .5 = 2x - 1).
>>> x = torch.tensor([0., .5, 1.]).reshape(1, 3, 1, 1)
tensor([0.0, 0.5, 1.0])

>>> t = T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
>>> t(x).flatten()
tensor([-1.,  0.,  1.])

How to properly use transforms.Normalize
In your case, you shouldn't use .5 as the mean and std parameters. This doesn't make any sense. If you're using a T.ToTensor prior to this you are essentially mapping your values to [-1, 1]...
Instead, you should initialize the transform with the actual dataset mean and std, the ones you measure from the dataset directly.
For instance, with Imagenet, the statistics are: [0.485, 0.456, 0.406] and [0.229, 0.224, 0.225] for mean and standard deviation respectively. You should be doing:
>>> t = T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))

Having imported torchvision.transforms as T


Instruction: How to insert values of one tensor into another?

Input:
I am trying to insert tensor y into the tensors x final dimension (y_dim). The final tensor should be of size (100, 16, 16, 1) where the values of y are placed in each of 100 x 0dimension
import torch

y_dim = 1
x = torch.randn(100, 16, 16, y_dim)
#OR x = torch.randn(100, 16, 16)
y = torch.randn(100)

Xy = torch.cat((x, y), dim=3)

Output:
I think you are missing something in your understanding of tensors and dimensions. The easiest thing is to consider your tensor x as a batch containing 100 maps of width and height 16, i.e. 100 16x16-maps. So you are manipulating a tensor containing 100*16*16 elements. Your y, on the other hand, contains 100 scalar values, it has 100 elements.
I'm turning the question back to you:

How would you concatenate 100 16x16-maps with 100 scalar values?

The above question has no answer. There are certain things that can be done though, assumptions that can be made on y in order to perform a concatenation:

If you had a tensor y containing 16x16 maps as well, then yes this operation would be achievable:
>>> x = torch.rand(100, 16, 16)
>>> y = torch.rand(100, 16, 16)

>>> torch.cat((x, y)).shape
torch.Size([200, 16, 16])

If you consider the y in your question, you could expand the 100 scalar values to 16x16 maps. And, then concatenate with x:
>>> x = torch.rand(100, 16, 16)
>>> y = torch.rand(100)

>>> y_r = y[:,  None, None].repeat(1, 16, 16)
>>> torch.cat((x, y_r))
torch.Size([200, 16, 16])


Instruction: Setting values of a tensor based on given indices of corresponding rows using pytorch

Input:
I've got a tensor A with shape (M, N), and have another tensor B with shape (M, P) and with values of given indices in corresponding rows of A. Now I would like to set the values of A with corresponding indices in B to 0.
For example:
In[1]: import torch
       A = torch.tensor([range(1,11), range(1,11), range(1,11)])
       A
Out[1]:
tensor([[ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10]])

In[2]: B = torch.tensor([[1,2], [2,3], [3,5]])
       B
Out[2]:
tensor([[1, 2],
        [2, 3],
        [3, 5]])

The objective is to set the value of the element with index 1,2 in the first row, 2,3 in the second row, and 3,5 in the third row of A to 0, i.e., setting A to
tensor([[ 1,  0,  0,  4,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  0,  0,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  3,  0,  5,  0,  7,  8,  9, 10]])

I have applied row by row for loop, and also tried scatter:
zeros = torch.zeros(A.shape, dtype=torch.float).to("cuda")
A = A.scatter_(1, B, zeros)

The two methods work fine, but all give quite poor performance. Actually, I infer that some efficient approach should exist based on an error before. I initially used A[:, B] = 0. This would set all the indices of appeared in B to 0, regardless of the row. However, the training speed improved drastically when doing A[:, B] = 0.
Is there any way to implement this more efficiently?

Output:
Here's what i would do:
import torch
A = torch.tensor([range(1,11), range(1,11), range(1,11)])
B = torch.tensor([[1,2], [2,3], [3,5]])
r, c = B.shape
idx0 = torch.arange(r).reshape(-1, 1).repeat(1, c).flatten()
idx1 = B.flatten()
A[idx0, idx1] = 0

output:
A =
tensor([[ 1,  0,  0,  4,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  0,  0,  5,  6,  7,  8,  9, 10],
        [ 1,  2,  3,  0,  5,  0,  7,  8,  9, 10]])


Instruction: CNN feature Extraction

Input:
class ResNet(nn.Module):
def __init__(self, output_features, fine_tuning=False):
    super(ResNet, self).__init__()
    self.resnet152 = tv.models.resnet152(pretrained=True)

    #freezing the feature extraction layers
    for param in self.resnet152.parameters():
        param.requires_grad = fine_tuning

    #self.features = self.resnet152.features

    self.num_fts = 512
    self.output_features = output_features

    # Linear layer goes from 512 to 1024
    self.classifier = nn.Linear(self.num_fts, self.output_features)
    nn.init.xavier_uniform_(self.classifier.weight)

    self.tanh = nn.Tanh()

def forward(self, x):

    h = self.resnet152(x)
    print('h:   ',h.shape)

    return h

image_model_resnet152=ResNet(output_features=10).to(device)
image_model_resnet152

Here, after printing the image_model_resnet152, I get:

Here, what is the difference between (avgpool): Linear(in_features=2048)
and (classifier): Linear(in_features=512)?
I am implementing an image captioning model, so which in_features should I take for an image?

Output:
ResNet is not as straightforward as VGG: it's not a sequential model, i.e. there is some model-specific logic inside the forward definition of the torchvision.models.resnet152, for instance, the flattening of features between the CNN and classifier. You can take a look at its source code.
The easiest thing to do in this case is to add a hook on the last layer of the CNN: layer4, and log the result of that layer in an external dict. This is done with register_forward_hook.
Define the hook:
out = {}
def result(module, input, output):
   out['layer4'] = output

Attach the hook on the submodule resnet.layer4:
>>> x = torch.rand(1,3,224,224)
>>> resnet = torchvision.models.resnet152()

>>> resnet.layer4.register_forward_hook(result)

After inference you will have access to the result inside of out:
>>> resnet(x)
>>> out['layer4']
(1, 2048, 7, 7)

You can look at another answer of mine on a more in-depth usage of forward hooks.

A possible implementation would be:
class NN(nn.Module):
    def __init__(self):
        super().__init__()
        self.resnet = torchvision.models.resnet152()
        self.resnet.layer4.register_forward_hook(result)
        self.out = {}

    @staticmethod
    def result(module, input, output):
        out['layer4'] = output

    def forward(self, x):
        x = self.resnet(x)
        return out['layer4']

You can then define additional layers for your custom classifier and call them inside forward.


Instruction: PyTorch resnet bad tensor dimensions

Input:
I'm trying to setup a face detection/recognition pipeline using Pytorch.
I load the image using opencv
image = cv2.imread('...')

I load the mtcnn face detection and resnet face recognition models
self.mtcnn = MTCNN(keep_all=True, device=self.device)
self.resnet = InceptionResnetV1(pretrained='vggface2').eval()

Then I run detection and recognition
cropped = detector.mtcnn(image)
detector.resnet(cropped.unsqueeze(0))

And get this error
Expected 4-dimensional input for 4-dimensional weight [32, 3, 3, 3], but got 5-dimensional input of size [1, 1, 3, 160, 160] instead

I also tried resizing the image to 512x512 and passing image_size=512 to the MTCNN constructor but I get a similar error.

Output:
I tried removing the unsqueeze and it worked
The guide I was following was using something other than opencv to load the image which probably returned the image as an array/tensor of images


Instruction: Pytorch permute not changing desired index

Input:
I am trying to use the permute function to swap the axis of my tensor but for some reason the output is not as expected. The output of the code is torch.Size([512, 256, 3, 3]) but I would expect it to be torch.Size([256, 512, 3, 3]). It doesn't look like I can use flip to switch 0, 1 index. Is there something i am missing? I wish to change the tensor such that the shape is (256, 512, 3, 3).
Reproducible code:
import torch

wtf = torch.rand(3, 3, 512, 256)
wtf = wtf.permute(2, 3, 1, 0)
print(wtf.shape)

Output:
The numbers provided to torch.permute are the indices of the axis in the order you want the new tensor to have.
Having set x as torch.rand(3, 3, 512, 256).

If you want to invert the order of axis: the initial order is 0, 1, 2, 3, you want 3, 2, 1, 0:
>>> wtf.permute(3, 2, 1, 0).shape
torch.Size([256, 512, 3, 3])

Inverting axis order is essentially the transpose operation:
>>> wtf.T
torch.Size([256, 512, 3, 3])

If you just want to invert and keep the order of the last two: original order is 0, 1, 2, 3 and resulting order is 3, 2, 0, 1:
>>> x.permute(3, 2, 0, 1).shape
torch.Size([256, 512, 3, 3])

The difference between the two options is that the last two axes of size 3 will be swapped.


Instruction: Confused about implementation of skip layers in CNN

Input:
I'm reading about AlphaGo Zero's network structure and came across this cheatsheet:

I'm having a hard time understanding how skip connections work dimensionally.
Specifically, it seems like each residual layer ends up with 2 stacked copies of the input it receives. Would this not cause the input size to grow exponentially with the depth of the network?
And could this be avoided by changing the output channel size of the conv2d filter? I see that in_C and out_C don't have to be the same in pytorch, but I don't know enough to understand the implications of these values being different.

Output:
With skip connection, you can indeed end up with twice the number of channels per connection. This is the case when you are concatenating the channels together. However, it doesn't necessarily have to grow exponentially, if you keep the number of output channels (what you refer to as out_C) under control.
For instance, if you have a skip connection providing a total of n channels and the convolutional layer gets in_C channels as input. Then you can define out_C as n as well, such that the resulting number of channels after concatenation is equal to 2*n. Ultimately, you decide on the number of output channels for each convolution, it is all about network capacity and how much it will be capable of learning.


Instruction: Will switching GPU device affect the gradient in PyTorch back propagation?

Input:
I use the Pytorch. In the computation, I move some data and operators A in the GPU. In the middle step, I move the data and operators B to CPU and continue the forward.
My question is that:
My operator B is very memory-consuming that cannot be used in GPU. Will this affect (some parts compute in GPU and the others are computed in CPU) the backpropagation?

Output:
Pytorch keeps track of the location of tensors. If you use .cpu() or .to('cpu') pytorch's native commands you should be okay.
See, e.g., this model parallel tutorial - the computation is split between two different GPU devices.


Instruction: Import transparent images to GAN

Input:
I have Images set which has transparency.
I'm trying to train GAN(Generative adversarial networks).
How can I preserve transparency. I can see from output images all transparent area is BLACK.
How can I avoid doing that ?
I think this is called "Alpha Channel".
Anyways How can I keep my transparency ?
Below is my code.
   # Importing the libraries
from __future__ import print_function
import torch.nn as nn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from generator import G
from discriminator import D
import os

batchSize = 64  # We set the size of the batch.
imageSize = 64  # We set the size of the generated images (64x64).
input_vector = 100
nb_epochs = 500
# Creating the transformations
transform = transforms.Compose([transforms.Resize((imageSize, imageSize)), transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5,
                                                                       0.5)), ])  # We create a list of transformations (scaling, tensor conversion, normalization) to apply to the input images.

# Loading the dataset
dataset = dset.ImageFolder(root='./data', transform=transform)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batchSize, shuffle=True,
                                         num_workers=2)  # We use dataLoader to get the images of the training set batch by batch.

# Defining the weights_init function that takes as input a neural network m and that will initialize all its weights.
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

def is_cuda_available():
    return torch.cuda.is_available()

def is_gpu_available():
    if is_cuda_available():
        if int(torch.cuda.device_count()) > 0:
            return True
        return False
    return False

# Create results directory
def create_dir(name):
    if not os.path.exists(name):
        os.makedirs(name)

# Creating the generator
netG = G(input_vector)
netG.apply(weights_init)

# Creating the discriminator
netD = D()
netD.apply(weights_init)

if is_gpu_available():
    netG.cuda()
    netD.cuda()

# Training the DCGANs

criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters(), lr=0.0002, betas=(0.5, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=0.0002, betas=(0.5, 0.999))

generator_model = 'generator_model'
discriminator_model = 'discriminator_model'

def save_model(epoch, model, optimizer, error, filepath, noise=None):
    if os.path.exists(filepath):
        os.remove(filepath)
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'loss': error,
        'noise': noise
    }, filepath)

def load_checkpoint(filepath):
    if os.path.exists(filepath):
        return torch.load(filepath)
    return None

def main():
    print("Device name : " + torch.cuda.get_device_name(0))
    for epoch in range(nb_epochs):

        for i, data in enumerate(dataloader, 0):
            checkpointG = load_checkpoint(generator_model)
            checkpointD = load_checkpoint(discriminator_model)
            if checkpointG:
                netG.load_state_dict(checkpointG['model_state_dict'])
                optimizerG.load_state_dict(checkpointG['optimizer_state_dict'])
            if checkpointD:
                netD.load_state_dict(checkpointD['model_state_dict'])
                optimizerD.load_state_dict(checkpointD['optimizer_state_dict'])

            # 1st Step: Updating the weights of the neural network of the discriminator

            netD.zero_grad()

            # Training the discriminator with a real image of the dataset
            real, _ = data
            if is_gpu_available():
                input = Variable(real.cuda()).cuda()
                target = Variable(torch.ones(input.size()[0]).cuda()).cuda()
            else:
                input = Variable(real)
                target = Variable(torch.ones(input.size()[0]))
            output = netD(input)
            errD_real = criterion(output, target)

            # Training the discriminator with a fake image generated by the generator
            if is_gpu_available():
                noise = Variable(torch.randn(input.size()[0], input_vector, 1, 1)).cuda()
                target = Variable(torch.zeros(input.size()[0])).cuda()
            else:
                noise = Variable(torch.randn(input.size()[0], input_vector, 1, 1))
                target = Variable(torch.zeros(input.size()[0]))
            fake = netG(noise)
            output = netD(fake.detach())
            errD_fake = criterion(output, target)

            # Backpropagating the total error
            errD = errD_real + errD_fake
            errD.backward()
            optimizerD.step()

            # 2nd Step: Updating the weights of the neural network of the generator
            netG.zero_grad()
            if is_gpu_available():
                target = Variable(torch.ones(input.size()[0])).cuda()
            else:
                target = Variable(torch.ones(input.size()[0]))
            output = netD(fake)
            errG = criterion(output, target)
            errG.backward()
            optimizerG.step()

            # 3rd Step: Printing the losses and saving the real images and the generated images of the minibatch every 100 steps

            print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f' % (epoch, nb_epochs, i, len(dataloader), errD.data, errG.data))
            save_model(epoch, netG, optimizerG, errG, generator_model, noise)
            save_model(epoch, netD, optimizerD, errD, discriminator_model, noise)

            if i % 100 == 0:
                create_dir('results')
                vutils.save_image(real, '%s/real_samples.png' % "./results", normalize=True)
                fake = netG(noise)
                vutils.save_image(fake.data, '%s/fake_samples_epoch_%03d.png' % ("./results", epoch), normalize=True)

if __name__ == "__main__":
    main()

generator.py
import torch.nn as nn
class G(nn.Module):
feature_maps = 512
kernel_size = 4
stride = 2
padding = 1
bias = False
def __init__(self, input_vector):
    super(G, self).__init__()
    self.main = nn.Sequential(
        nn.ConvTranspose2d(input_vector, self.feature_maps, self.kernel_size, 1, 0, bias=self.bias),
        nn.BatchNorm2d(self.feature_maps), nn.ReLU(True),
        nn.ConvTranspose2d(self.feature_maps, int(self.feature_maps // 2), self.kernel_size, self.stride, self.padding,
                           bias=self.bias),
        nn.BatchNorm2d(int(self.feature_maps // 2)), nn.ReLU(True),
        nn.ConvTranspose2d(int(self.feature_maps // 2), int((self.feature_maps // 2) // 2), self.kernel_size, self.stride,
                           self.padding,
                           bias=self.bias),
        nn.BatchNorm2d(int((self.feature_maps // 2) // 2)), nn.ReLU(True),
        nn.ConvTranspose2d((int((self.feature_maps // 2) // 2)), int(((self.feature_maps // 2) // 2) // 2), self.kernel_size,
                           self.stride, self.padding,
                           bias=self.bias),
        nn.BatchNorm2d(int((self.feature_maps // 2) // 2) // 2), nn.ReLU(True),
        nn.ConvTranspose2d(int(((self.feature_maps // 2) // 2) // 2), 4, self.kernel_size, self.stride, self.padding,
                           bias=self.bias),
        nn.Tanh()
    )

def forward(self, input):
    output = self.main(input)
    return output

discriminator.py
import torch.nn as nn
class D(nn.Module):
    feature_maps = 64
    kernel_size = 4
    stride = 2
    padding = 1
    bias = False
    inplace = True

    def __init__(self):
        super(D, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(4, self.feature_maps, self.kernel_size, self.stride, self.padding, bias=self.bias),
            nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps, self.feature_maps * 2, self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * 2), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * 2, self.feature_maps * (2 * 2), self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2), self.feature_maps * (2 * 2 * 2), self.kernel_size, self.stride,
                      self.padding, bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2 * 2), 1, self.kernel_size, 1, 0, bias=self.bias),
            nn.Sigmoid()
        )

    def forward(self, input):
        output = self.main(input)
        return output.view(-1)

Output:
Using dset.ImageFolder, without explicitly defining the function that reads the image (the loader) results with your dataset using the default pil_loader:
def pil_loader(path: str) -> Image.Image:
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGB')

As you can see, the default loader discards the alpha channel and forces the image to be with only three color channels: RGB.
You can define your own loader:
def pil_loader_rgba(path: str) -> Image.Image:
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGBA')  # force alpha channel

You can use this loader in your dataset:
dataset = dset.ImageFolder(root='./data', transform=transform, loader=pil_loader_rgba)

Now your images will have the alpha channel.
Note that the transparency ("alpha channel") is an additional channel and is not part of the RGB channels. You need to make sure your model knows how to handle 4-channel inputs, otherwise, you'll run into errors such as this.


Instruction: pytorch indexing for pixel probabilities

Input:
Suppose I need to classify each pixel into one of 3 classes. I wish to get the probability of each pixel. Here is a minimal example. Question is, how do I get those probabilities.
import torch
import torch.nn.functional as F

y = torch.randint(0, 3, (2, 1, 5, 5)) # classes
logits = torch.randn(2, 3, 5, 5)
prob  = F.softmax(logits, dim=1) # probability map

prob[y] # does not work

Output:
You are looking for torch.gather:
torch.gather(prob,1, y)

You gather the probabilities along the first dimension according to the indices of y.


Instruction: Pytorch Lightning Automatic Logging - AttributeError: 'NoneType' object has no attribute '_results'

Input:
Unable to use Automatic Logging (self.log) when calling training_step() on Pytorch Lightning, what am I missing? Here is a minimal example:
import pytorch_lightning as pl
import torch
import torch.nn as nn
import torch.nn.functional as F

class LitModel(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.l1 = nn.Linear(100, 4)

    def forward(self, x):
        return torch.relu(self.l1(x.view(x.size(0), -1)))

    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = F.cross_entropy(y_hat, y.long())
        self.log("train_loss", loss) # <-- error
        return loss

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.02)

pl_model = LitModel()
x = torch.rand((10,100))
y = torch.randint(0,4, size=(10,))
batch = (x,y)
loss = pl_model.training_step(batch, 0)

Error:
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-34-b9419bfca30f> in <module>
     25 y = torch.randint(0,4, size=(10,))
     26 batch = (x,y)
---> 27 loss = pl_model.training_step(batch, 0)

<ipython-input-34-b9419bfca30f> in training_step(self, batch, batch_idx)
     14         y_hat = self(x)
     15         loss = F.cross_entropy(y_hat, y.long())
---> 16         self.log("train_loss", loss)
     17         return loss
     18

D:\programs\anaconda3\lib\site-packages\pytorch_lightning\core\lightning.py in log(self, name, value, prog_bar, logger, on_step, on_epoch, reduce_fx, tbptt_reduce_fx, tbptt_pad_token, enable_graph, sync_dist, sync_dist_op, sync_dist_group, add_dataloader_idx, batch_size, metric_attribute, rank_zero_only)
    405         on_epoch = self.__auto_choose_log_on_epoch(on_epoch)
    406
--> 407         results = self.trainer._results
    408         assert results is not None
    409         assert self._current_fx_name is not None

AttributeError: 'NoneType' object has no attribute '_results'

Output:
This is NOT the correct usage of LightningModule class. You can't call a hook (namely .training_step()) manually and expect everything to work fine.
You need to setup a Trainer as suggested by PyTorch Lightning at the very start of its tutorial - it is a requirement. The functions (or hooks) that you define in a LightningModule merely tells Lightning "what to do" in a specific situation (in this case, at each training step). It is the Trainer that actually "orchestrates" the training by instantiating the necessary environment (including Logging functionality) and feeding it into the Lightning Module whenever needed.
So, do it the way Lightning suggests and it will work.


Instruction: Cannot install the gpu version of torch and torchvision in poetry due to a dependency problem

Input:
I am trying to create a virtual environment for machine learning using poetry. So, I am using pytorch as a framework for deep learning. I will extract the relevant part of my pyproject.toml.
[tool.poetry.dependencies].
python = "^3.8"
torch = { url = "https://download.pytorch.org/whl/cu111/torch-1.8.0%2Bcu111-cp38-cp38-linux_x86_64.whl"}
torchvision = { url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.0%2Bcu111-cp38-cp38-linux_x86_64.whl" }

Since pytroch uses the GPU, you need to install it by specifying the whl file. If you install it this way, the version of pytroch will be 1.8.0+cu111. torchvision corresponding to 1.8.0 of pytorch is 0.9.0. The version of pytroch that this torchvision depends on is 1.8.0 (without cu111). Therefore, I cannot create a virtual environment using poetry with the following error.

  SolverProblemError

  Because torchvision (0.9.0) depends on torch (1.8.0)
   and mdwithpriorenergy depends on torch (1.8.0+cu111), torchvision is forbidden.
  So, because mdwithpriorenergy depends on torchvision (0.9.0), version solving failed.

So, because [env name] depends on torchvision (0.9.0), version solving failed.

I also made the following changes to torchvision in pyproject.toml above, but they did not work.
[tool.poetry.dependencies].
python = "^3.8"
torch = { url = "https://download.pytorch.org/whl/cu111/torch-1.8.0%2Bcu111-cp38-cp38-linux_x86_64.whl"}
- torchvision = { url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.0%2Bcu111-cp38-cp38-linux_x86_64.whl"}
+ torchvision = "*"

In this case, I received the following error
  AttributeError

  'EmptyConstraint' object has no attribute 'allows'.

Please tell me how to solve this error.

Output:
As far as I know, this does not yet supported in poetry (without ugly hacks), see Issue 2613.
That said, there is a fork I am maintaining called relaxed-poetry. It is a very young fork but it supports what you want with the following configuration:
[tool.poetry.dependencies]
python = "^3.8"
torch = { version = "=1.8.0+cu111", source = "pytorch" }
torchvision = { version = "=0.9.0+cu111", source = "pytorch" }

[[tool.poetry.source]]
name = "pytorch"
url = "https://download.pytorch.org/whl/cu111/"
secondary = true

If you think it can help you, you can install it side by side with poetry and use the command rp instead
Do note though that the installation is going to take some time as this is a large dependency and a relatively slow source..


Instruction: Importing Transparent images gives RuntimeError: The size of tensor a (4) must match the size of tensor b (3) at non-singleton dimension 0

Input:
I'm trying to learn AI.
I have GAN (generative adversarial network) code with images with ALPHA Channel(transparency).
All images have alpha channel.
To prove that I wrote small image_validator.py program like below
from PIL import Image
import glob

def main():
    image_list = []
    img_number = 0
    for filename in glob.glob('data/*/*.*'):
        try:
            im = Image.open(filename)
            # print(filename)
            if str(im.mode) != "RGBA":
                print("alpha " + str(im.mode))
                img_number = img_number+1
                print(str(img_number))
        except Exception as e:
            print("Error : "+filename)

if __name__ == "__main__":
    main()

Above program prints nothing which means all images have ALPHA Channel. To Test above program I added single image WITHOUT ALPHA Channel. So I can confirm all images have ALPHA CHANNEL.
my generator.py is like below
import torch.nn as nn
class G(nn.Module):
    feature_maps = 512
    kernel_size = 4
    stride = 2
    padding = 1
    bias = False

    def __init__(self, input_vector):
        super(G, self).__init__()
        self.main = nn.Sequential(
            nn.ConvTranspose2d(input_vector, self.feature_maps, self.kernel_size, 1, 0, bias=self.bias),
            nn.BatchNorm2d(self.feature_maps), nn.ReLU(True),
            nn.ConvTranspose2d(self.feature_maps, int(self.feature_maps // 2), self.kernel_size, self.stride, self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int(self.feature_maps // 2)), nn.ReLU(True),
            nn.ConvTranspose2d(int(self.feature_maps // 2), int((self.feature_maps // 2) // 2), self.kernel_size, self.stride,
                               self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int((self.feature_maps // 2) // 2)), nn.ReLU(True),
            nn.ConvTranspose2d((int((self.feature_maps // 2) // 2)), int(((self.feature_maps // 2) // 2) // 2), self.kernel_size,
                               self.stride, self.padding,
                               bias=self.bias),
            nn.BatchNorm2d(int((self.feature_maps // 2) // 2) // 2), nn.ReLU(True),
            nn.ConvTranspose2d(int(((self.feature_maps // 2) // 2) // 2), 4, self.kernel_size, self.stride, self.padding,
                               bias=self.bias),
            nn.Tanh()
        )

    def forward(self, input):
        output = self.main(input)
        return output

my discriminator.py is like below
    import torch.nn as nn
class D(nn.Module):
    feature_maps = 64
    kernel_size = 4
    stride = 2
    padding = 1
    bias = False
    inplace = True

    def __init__(self):
        super(D, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(4, self.feature_maps, self.kernel_size, self.stride, self.padding, bias=self.bias),
            nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps, self.feature_maps * 2, self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * 2), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * 2, self.feature_maps * (2 * 2), self.kernel_size, self.stride, self.padding,
                      bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2), self.feature_maps * (2 * 2 * 2), self.kernel_size, self.stride,
                      self.padding, bias=self.bias),
            nn.BatchNorm2d(self.feature_maps * (2 * 2 * 2)), nn.LeakyReLU(0.2, inplace=self.inplace),
            nn.Conv2d(self.feature_maps * (2 * 2 * 2), 1, self.kernel_size, 1, 0, bias=self.bias),
            nn.Sigmoid()
        )

    def forward(self, input):
        output = self.main(input)
        return output.view(-1)

my main program gan.py is like below
    # Importing the libraries
from __future__ import print_function
import torch.nn as nn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from generator import G
from discriminator import D
import os
from PIL import Image

batchSize = 64  # We set the size of the batch.
imageSize = 64  # We set the size of the generated images (64x64).
input_vector = 100
nb_epochs = 500
# Creating the transformations
transform = transforms.Compose([transforms.Resize((imageSize, imageSize)), transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5,
                                                                       0.5)), ])  # We create a list of transformations (scaling, tensor conversion, normalization) to apply to the input images.

def pil_loader_rgba(path: str) -> Image.Image:
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGBA')

# Loading the dataset
dataset = dset.ImageFolder(root='./data', transform=transform, loader=pil_loader_rgba)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batchSize, shuffle=True,
                                         num_workers=2)  # We use dataLoader to get the images of the training set batch by batch.

# Defining the weights_init function that takes as input a neural network m and that will initialize all its weights.
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

def is_cuda_available():
    return torch.cuda.is_available()

def is_gpu_available():
    if is_cuda_available():
        if int(torch.cuda.device_count()) > 0:
            return True
        return False
    return False

# Create results directory
def create_dir(name):
    if not os.path.exists(name):
        os.makedirs(name)

# Creating the generator
netG = G(input_vector)
netG.apply(weights_init)

# Creating the discriminator
netD = D()
netD.apply(weights_init)

if is_gpu_available():
    netG.cuda()
    netD.cuda()

# Training the DCGANs

criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters(), lr=0.0002, betas=(0.5, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=0.0002, betas=(0.5, 0.999))

generator_model = 'generator_model'
discriminator_model = 'discriminator_model'

def save_model(epoch, model, optimizer, error, filepath, noise=None):
    if os.path.exists(filepath):
        os.remove(filepath)
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'loss': error,
        'noise': noise
    }, filepath)

def load_checkpoint(filepath):
    if os.path.exists(filepath):
        return torch.load(filepath)
    return None

def main():
    print("Device name : " + torch.cuda.get_device_name(0))
    for epoch in range(nb_epochs):

        for i, data in enumerate(dataloader, 0):
            checkpointG = load_checkpoint(generator_model)
            checkpointD = load_checkpoint(discriminator_model)
            if checkpointG:
                print("checkpointG")
                netG.load_state_dict(checkpointG['model_state_dict'])
                optimizerG.load_state_dict(checkpointG['optimizer_state_dict'])
            if checkpointD:
                netD.load_state_dict(checkpointD['model_state_dict'])
                optimizerD.load_state_dict(checkpointD['optimizer_state_dict'])

            # 1st Step: Updating the weights of the neural network of the discriminator

            netD.zero_grad()

            # Training the discriminator with a real image of the dataset
            real, _ = data
            if is_gpu_available():
                print("True")
                input = Variable(real.cuda()).cuda()
                target = Variable(torch.ones(input.size()[0]).cuda()).cuda()
            else:
                input = Variable(real)
                target = Variable(torch.ones(input.size()[0]))
            output = netD(input)
            errD_real = criterion(output, target)

            # Training the discriminator with a fake image generated by the generator
            if is_gpu_available():
                noise = Variable(torch.randn(input.size()[0], input_vector, 1, 1)).cuda()
                target = Variable(torch.zeros(input.size()[0])).cuda()
            else:
                noise = Variable(torch.randn(input.size()[0], input_vector, 1, 1))
                target = Variable(torch.zeros(input.size()[0]))
            fake = netG(noise)
            output = netD(fake.detach())
            errD_fake = criterion(output, target)

            # Backpropagating the total error
            errD = errD_real + errD_fake
            errD.backward()
            optimizerD.step()

            # 2nd Step: Updating the weights of the neural network of the generator
            netG.zero_grad()
            if is_gpu_available():
                target = Variable(torch.ones(input.size()[0])).cuda()
            else:
                target = Variable(torch.ones(input.size()[0]))
            output = netD(fake)
            errG = criterion(output, target)
            errG.backward()
            optimizerG.step()

            # 3rd Step: Printing the losses and saving the real images and the generated images of the minibatch every 100 steps

            print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f' % (
            epoch, nb_epochs, i, len(dataloader), errD.data, errG.data))
            save_model(epoch, netG, optimizerG, errG, generator_model, noise)
            save_model(epoch, netD, optimizerD, errD, discriminator_model, noise)

            if i % 100 == 0:
                create_dir('results')
                vutils.save_image(real, '%s/real_samples.png' % "./results", normalize=True)
                fake = netG(noise)
                vutils.save_image(fake.data, '%s/fake_samples_epoch_%03d.png' % ("./results", epoch), normalize=True)

if __name__ == "__main__":
    main()

But when I run my program I'm getting this error
Traceback (most recent call last):
  File ".\gans.py", line 178, in <module>
    main()
  File ".\gans.py", line 109, in main
    for i, data in enumerate(dataloader, 0):
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 521, in __next__
    data = self._next_data()
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 1203, in _next_data
    return self._process_data(data)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\dataloader.py", line 1229, in _process_data
    data.reraise()
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\_utils.py", line 425, in reraise
    raise self.exc_type(msg)
RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\_utils\worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "C:\Users\Akila\anaconda3\lib\site-packages\torchvision\datasets\folder.py", line 234, in __getitem__
    sample = self.transform(sample)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torchvision\transforms\transforms.py", line 60, in __call__
    img = t(img)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torchvision\transforms\transforms.py", line 221, in forward
    return F.normalize(tensor, self.mean, self.std, self.inplace)
  File "C:\Users\Akila\anaconda3\lib\site-packages\torchvision\transforms\functional.py", line 335, in normalize
    tensor.sub_(mean).div_(std)
RuntimeError: The size of tensor a (4) must match the size of tensor b (3) at non-singleton dimension 0

I did a debugging and I found the issue is with this line
for i, data in enumerate(dataloader, 0):

IF I change this line --> return img.convert('RGBA')
TO this --> return img.convert('RGB')
program works fine.
But I can guarantee all of images have alpha channel.
Because my image_validator.py program prints nothing
I even tried running my main program with SINGLE image which has alpha channel still gives same error.
What am I doing wrong ?
How can I preserve transparency of my Images ?
I do NOT want to lose my transparency.

Output:
Regarding the error message

RuntimeError: The size of tensor a (4) must match the size of tensor b (3) at non-singleton dimension 0

would lead to suggest that there's a problem with this call: sample = self.transform(sample)
Indeed, the issue is you are using a T.Normalize transform which only expects three channels (you specified a mean and std for three channels only, not four).
transform = transforms.Compose([
   transforms.Resize((imageSize, imageSize)),
   transforms.ToTensor(),
   transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5,0.5))])

Instead, you should provide a four-element tuple for both arguments. For example (this is an example, this might run but won't necessarily make sense... see explanation below):
    transforms.Normalize((0.5, 0.5, 0.5, 0.5), (0.5, 0.5,0.5, 0.5))])

Other than that, I should ask: do you know why you are using .5 for both parameters of mean and std? If not, chances are you are not using it properly. Please read about it on this answer and its applications.