Instruction: Weights & Biases sweep cannot import modules with pytorch lightning

Input:
I am training a variational autoencoder, using pytorch-lightning. My pytorch-lightning code works with a Weights and Biases logger. I am trying to do a parameter sweep using a W&B parameter sweep.
The hyperparameter search procedure is based on what I followed from this repo.
The runs initialise correctly, but when my training script is run with the first set of hyperparameters, i get the following error:
2020-08-14 14:09:07,109 - wandb.wandb_agent - INFO - About to run command: /usr/bin/env python train_sweep.py --LR=0.02537477586974176
Traceback (most recent call last):
  File "train_sweep.py", line 1, in <module>
    import yaml
ImportError: No module named yaml

yaml is installed and is working correctly. I can train the network by setting the parameters manually, but not with the parameter sweep.
Here is my sweep script to train the VAE:
import yaml
import numpy as np
import ipdb
import torch
from vae_experiment import VAEXperiment
import torch.backends.cudnn as cudnn
from pytorch_lightning import Trainer
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.callbacks import EarlyStopping
from vae_network import VanillaVAE
import os
import wandb
from utils import get_config, log_to_wandb

# Sweep parameters
hyperparameter_defaults = dict(
    root='data_semantics',
    gpus=1,
    batch_size = 2,
    lr = 1e-3,
    num_layers = 5,
    features_start = 64,
    bilinear = False,
    grad_batches = 1,
    epochs = 20
)

wandb.init(config=hyperparameter_defaults)
config = wandb.config

def main(hparams):

    model = VanillaVAE(hparams['exp_params']['img_size'], **hparams['model_params'])
    model.build_layers()
    experiment = VAEXperiment(model, hparams['exp_params'], hparams['parameters'])

    logger = WandbLogger(
        project='vae',
        name=config['logging_params']['name'],
        version=config['logging_params']['version'],
        save_dir=config['logging_params']['save_dir']
        )

    wandb_logger.watch(model.net)

    early_stopping = EarlyStopping(
       monitor='val_loss',
       min_delta=0.00,
       patience=3,
       verbose=False,
       mode='min'
    )

    runner = Trainer(weights_save_path="../../Logs/",
     min_epochs=1,
     logger=logger,
     log_save_interval=10,
     train_percent_check=1.,
     val_percent_check=1.,
     num_sanity_val_steps=5,
     early_stop_callback = early_stopping,
     **config['trainer_params']
     )

    runner.fit(experiment)

if __name__ == '__main__':
    main(config)

Why am I getting this error?

Output:
The problem is that the structure of my code and the way that I was running the wandb commands was not in the correct order. Looking at this pytorch-ligthning with wandb is the correct structure to follow.
Here is my refactored code:
#!/usr/bin/env python
import wandb
from utils import get_config

#---------------------------------------------------------------------------------------------

def main():

    """
    The training function used in each sweep of the model.
    For every sweep, this function will be executed as if it is a script on its own.
    """

    import wandb
    import yaml
    import numpy as np
    import torch
    from vae_experiment import VAEXperiment
    import torch.backends.cudnn as cudnn
    from pytorch_lightning import Trainer
    from pytorch_lightning.loggers import WandbLogger
    from pytorch_lightning.callbacks import EarlyStopping
    from vae_network import VanillaVAE
    import os
    from utils import log_to_wandb, format_config

    path_to_config = 'sweep.yaml'
    config = get_config(path_to_yaml)

    path_to_defaults = 'defaults.yaml'
    param_defaults = get_config(path_to_defaults)

    wandb.init(config=param_defaults)

    config = format_config(config, wandb.config)
    model = VanillaVAE(config['meta']['img_size'], hidden_dims = config['hidden_dims'], latent_dim  = config['latent_dim'])
    model.build_layers()

    experiment = VAEXperiment(model, config)

    early_stopping = EarlyStopping(
       monitor='val_loss',
       min_delta=0.00,
       patience=3,
       verbose=False,
       mode='max'
    )

    runner = Trainer(weights_save_path=config['meta']['save_dir'],
        min_epochs=1,
        train_percent_check=1.,
        val_percent_check=1.,
        num_sanity_val_steps=5,
        early_stop_callback = early_stopping,
        **config['trainer_params'])

    runner.fit(experiment)
    log_to_wandb(config, runner, experiment, path_to_config)

#---------------------------------------------------------------------------------------------

path_to_yaml = 'sweep.yaml'
sweep_config = get_config(path_to_yaml)
sweep_id = wandb.sweep(sweep_config)
wandb.agent(sweep_id, function=main)

#---------------------------------------------------------------------------------------------


Instruction: tensor transformation in pytorch?

Input:
I have a tensor of shape (size, 1) and I want to convert it into of shape (size, lookback, 1) by shifting its values. A pandas equivalent is below
size = 7
lookback = 3

data = pd.DataFrame(np.arange(size), columns=['out'])  # input
y = np.full((len(data), lookback, 1), np.nan)          # required/output
for j in range(lookback):
    y[:, j, 0] = data['out'].shift(lookback - j - 1).fillna(method="bfill")

How can I acheive similar in pytorch?
Example input:
[0, 1, 2, 3, 4, 5, 6]

Desired output:
[[0. 0. 0.]
 [0. 0. 1.]
 [0. 1. 2.]
 [1. 2. 3.]
 [2. 3. 4.]
 [3. 4. 5.]
 [4. 5. 6.]]

Output:
You can use Tensor.unfold for this. First though you will need to pad the front of the tensor, for that you could use nn.functional.pad. E.g.
import torch
import torch.nn.functional as F

size = 7
loopback = 3

data = torch.arange(size, dtype=torch.float)

# pad front of data with 2 values
# replicate padding requires 3d, 4d, or 5d tensor, hence the creation of two unitary dimensions before padding
data_padded = F.pad(data[None, None, ...], (loopback - 1, 0), 'replicate')[0, 0, ...]
# unfold with window size of 3 with step size of 1
y = data_padded.unfold(dimension=0, size=loopback, step=1)

which gives output of
tensor([[0., 0., 0.],
        [0., 0., 1.],
        [0., 1., 2.],
        [1., 2., 3.],
        [2., 3., 4.],
        [3., 4., 5.],
        [4., 5., 6.]])


Instruction: How to solve "NotImplementedError"

Input:
I defined a three layer convolution layer(self.convs) ,the input tensor has the shape([100,10,24])
x_convs = self.convs(Variable(torch.from_numpy(X).type(torch.FloatTensor)))

>>Variable(torch.from_numpy(X).type(torch.FloatTensor)).shape

torch.Size([100, 10, 24])

>>self.convs

ModuleList(
  (0): ConvBlock(
    (conv): Conv1d(24, 8, kernel_size=(5,), stride=(1,), padding=(2,))
    (relu): ReLU()
    (maxpool): AdaptiveMaxPool1d(output_size=10)
    (zp): ConstantPad1d(padding=(1, 0), value=0)
  )
  (1): ConvBlock(
    (conv): Conv1d(8, 8, kernel_size=(5,), stride=(1,), padding=(2,))
    (relu): ReLU()
    (maxpool): AdaptiveMaxPool1d(output_size=10)
    (zp): ConstantPad1d(padding=(1, 0), value=0)
  )
  (2): ConvBlock(
    (conv): Conv1d(8, 8, kernel_size=(5,), stride=(1,), padding=(2,))
    (relu): ReLU()
    (maxpool): AdaptiveMaxPool1d(output_size=10)
    (zp): ConstantPad1d(padding=(1, 0), value=0)
  )
)

When I excuate x_convs = self.convs(Variable(torch.from_numpy(X).type(torch.FloatTensor))), it gives me the error
    `94             registered hooks while the latter silently ignores them.

     95         """

---> 96         raise NotImplementedError`

The ConvBlock is defined as below
class ConvBlock(nn.Module):
    def __init__(self, T, in_channels, out_channels, filter_size):
        super(ConvBlock, self).__init__()
        padding = self._calc_padding(T, filter_size)
        self.conv=nn.Conv1d(in_channels, out_channels, filter_size, padding=padding)
        self.relu=nn.ReLU()
        self.maxpool=nn.AdaptiveMaxPool1d(T)
        self.zp=nn.ConstantPad1d((1, 0), 0)

    def _calc_padding(self, Lin, kernel_size, stride=1, dilation=1):
        p = int(((Lin-1)*stride + 1 + dilation*(kernel_size - 1) - Lin)/2)
        return p

    def forward(self, x):
        x = x.permute(0,2,1)
        x = self.conv(x)
        x = self.relu(x)
        x = self.maxpool(x)
        x = x.permute(0,2,1)
        return x

The "forward" function has correct indent, so I cannot figure it out what is going on.

Output:
You are trying to call a ModuleList, which is a list (i.e. a list object in Python), slightly modified for being used with PyTorch.
A quick fix would be to call the self.convs as:
x_convs = self.convs[0](Variable(torch.from_numpy(X).type(torch.FloatTensor)))

if len(self.convs) > 1:
    for conv in self.convs[1:]:
        x_convs = conv(x_convs)

That is, although self.convs is a list, each member of it is a Module. You can directly call each member of the  self.convs, using its index, e.g. ``self.convsan_index`.
Or, you can do it with the help of functools module:
from functools import reduce

def apply_layer(layer_input, layer):
    return layer(layer_input)

output_of_self_convs = reduce(apply_layer, self.convs, Variable(torch.from_numpy(X).type(torch.FloatTensor)))

P.S. Though, the Variable keyword is not used anymore.


Instruction: PyTorch Input and hidden tensors not on the same device

Input:
I'm creating a simple LSTM model to predict some sales data. I am trying to train it on a GPU, but there seems to be a problem with casting the hidden state tensor to cuda.
I get the following error:
RuntimeError: Input and hidden tensors are not at the same device, found input tensor at cuda:0 and hidden tensor at cpu.
How can I train the model on a GPU? I cast the training data, initial hidden states, and the model to cuda, yet I still get the error.
Here's my code:
# Convert train_norm from an array to a tensor
train_norm = torch.FloatTensor(train_norm).view(-1).cuda()

# define a window size
window_size = 12

# Define function to create seq/label tuples

def input_data(seq, ws): # ws is window size
    out = []
    L = len(seq)

    for i in range(L-ws):
        window = seq[i:i+ws]
        label = seq[i+ws:i+ws+1]

        out.append((window, label))

    return out

# Apply the input_data function to the train_norm
train_data = input_data(train_norm, window_size)

class LSTM(nn.Module):
    def __init__(self, input_size=1, hidden_size=100, output_size=1):
        super().__init__()
        self.hidden_size = hidden_size

        # Add an LSTM layer:
        self.lstm = nn.LSTM(input_size, hidden_size)

        # Add a fully connected linear layer:
        self.linear = nn.Linear(hidden_size, output_size)

        # Initialize h0 and c0:
        self.hidden = (torch.zeros(1, 1, hidden_size).cuda(), torch.zeros(1, 1, hidden_size).cuda())

    def forward(self, seq):
        lstm_out, self.hidden = self.lstm(seq.view(len(seq), 1, -1), self.hidden)
        pred = self.linear(lstm_out.view(len(seq), -1))

        return pred[-1] # get only the last value

model = LSTM().cuda()

criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

epochs = 200

import time
start_time = time.time()

for epoch in range(epochs):

    # Extract the sequence and label from the training data
    for seq, y_train in train_data:

        # Reset the parameters and hidden states
        optimizer.zero_grad()
        hidden = (torch.zeros(1, 1, model.hidden_size),
                  torch.zeros(1, 1, model.hidden_size))
        model.hidden = hidden

        # Predict the values
        y_pred = model(seq)

        # Calculate loss and perform backpropagation
        loss = criterion(y_pred, y_train)
        loss.backward()
        optimizer.step()

    print(f'epoch: {epoch+1:2} loss: {loss.item():10.8f}')

print(f'Training took {time.time() - start_time:.0f} seconds')

Output:
First of all you are initializing hidden when there is absolutely no point to do it. If hidden isn't passed to LSTM layer it will be zero by default, please see documentation. This gives us the following model:
class LSTM(nn.Module):
    def __init__(self, input_size=1, hidden_size=100, output_size=1):
        super().__init__()
        self.hidden_size = hidden_size

        # Add an LSTM layer:
        self.lstm = nn.LSTM(input_size, hidden_size)

        # Add a fully connected linear layer:
        self.linear = nn.Linear(hidden_size, output_size)

    def forward(self, seq):
        lstm_out, _ = self.lstm(seq.view(len(seq), 1, -1))
        return self.linear(lstm_out.view(len(seq), -1))

Your pred[-1] is probably wrong as well as you are only returning the last element of batch from linear layer...
Also your training should be this (see hidden removed and added cuda to seq and y_train):
for epoch in range(epochs):

    # Extract the sequence and label from the training data
    for seq, y_train in train_data:

        # Reset the parameters and hidden states
        optimizer.zero_grad()

        # Predict the values
        # Add cuda to sequence
        y_pred = model(seq.cuda())

        # Calculate loss and perform backpropagation
        loss = criterion(y_pred, y_train.cuda())
        loss.backward()
        optimizer.step()

    print(f'epoch: {epoch+1:2} loss: {loss.item():10.8f}')

print(f'Training took {time.time() - start_time:.0f} seconds')

This alleviates problems with cuda (it's not a solution to hardcode it everywhere you possibly can...) and makes your code more readable.


Instruction: best way of tqdm for data loader

Input:
How to use tqdm for data_loader ?
is this the correct way?
for i,j in enumerate(data_loader,total = 100):
           pass

Output:
You need to wrap the iterable with tqdm, as their documentation clearly says:

Instantly make your loops show a smart progress meter - just wrap any
iterable with tqdm(iterable), and you’re done!

If you're enumerating over an iterable, you can do something like the following. Sleep is only for visualizing it.
from tqdm import tqdm
from time import sleep

data_loader = list(range(1000))

for i, j in enumerate(tqdm(data_loader)):
    sleep(0.01)


Instruction: torch.save() gives : RuntimeError: CUDA error: no CUDA-capable device is detected

Input:
I was training a neural network model on GPU but I get the above mentioned error when I use torch.save() to save checkpoints. My question is even though I have a CUDA device why am I getting the mentioned error? My model was running okay on the GPU please see bellow the Output of: nvidia-smi command.
$ nvidia-smi
Sat Aug 15 09:51:58 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.100      Driver Version: 440.100      CUDA Version: 10.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce RTX 2060    Off  | 00000000:01:00.0 Off |                  N/A |
| N/A   55C    P3    33W /  N/A |   4774MiB /  5934MiB |     97%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0      7080      C   python3                                     4763MiB |
+-----------------------------------------------------------------------------+

$ python --version
Python 3.8.2

$ python -c "import torch; print(torch.__version__)"
1.5.1

$ python -c "import torchvision as torch; print(torch.__version__)"
0.6.1

I have even tried the following:

os.environ["CUDA_VISIBLE_DEVICES"] = '0'

torch.save({
                'epoch': epoch + 1,
                'metrics': metrics,
                'model': model.state_dict(),
                'optimizer' : optimizer.state_dict(),
            }, name)

But nothing worked. I am new to deep learning and still learning PyTorch. Please do pardon my ignorance.

Output:
It seems some other problem. Test this toy example and see if the problem is in torch.save() or any other command.
import torch
import torchvision

model = torchvision.models.resnet18()
model = model.cuda()

torch.save(model.state_dict(), 'net')


Instruction: How does torch.argmax work for 4-dimensions

Input:
I am a newbie in Pytorch. Even though I read the documentation, it is unclear for me how does torch.argmax() applied to first dimension work when we have 4-dimensional input. Also, how does keepdims=True change the output?
Here is an example of each case:
k = torch.rand(2, 3, 4, 4)
print(k):

tensor([[[[0.2912, 0.4818, 0.1123, 0.3196],
          [0.6606, 0.1547, 0.0368, 0.9475],
          [0.4753, 0.7428, 0.5931, 0.3615],
          [0.6729, 0.7069, 0.1569, 0.3086]],

         [[0.6603, 0.7777, 0.3546, 0.2850],
          [0.3681, 0.5295, 0.8812, 0.6093],
          [0.9165, 0.2842, 0.0260, 0.1768],
          [0.9371, 0.9889, 0.6936, 0.7018]],

         [[0.5880, 0.0349, 0.0419, 0.3913],
          [0.5884, 0.9408, 0.1707, 0.1893],
          [0.3260, 0.4410, 0.6369, 0.7331],
          [0.9448, 0.7130, 0.3914, 0.2775]]],

        [[[0.9433, 0.8610, 0.9936, 0.1314],
          [0.8627, 0.3103, 0.3066, 0.3547],
          [0.3396, 0.1892, 0.0385, 0.5542],
          [0.4943, 0.0256, 0.7875, 0.5562]],

         [[0.2338, 0.2498, 0.4749, 0.2520],
          [0.4405, 0.1605, 0.6219, 0.8955],
          [0.2326, 0.1816, 0.5032, 0.8732],
          [0.2089, 0.6131, 0.1898, 0.0517]],

         [[0.1472, 0.8059, 0.6958, 0.9047],
          [0.6403, 0.2875, 0.5746, 0.5908],
          [0.8668, 0.4602, 0.8224, 0.9307],
          [0.2077, 0.5665, 0.8671, 0.4365]]]])

argmax = torch.argmax(k, axis=1)
print(argmax):
tensor([[[1, 1, 1, 2],
         [0, 2, 1, 0],
         [1, 0, 2, 2],
         [2, 1, 1, 1]],

        [[0, 0, 0, 2],
         [0, 0, 1, 1],
         [2, 2, 2, 2],
         [0, 1, 2, 0]]])

argmax = torch.argmax(k, axis=1, keepdims=True)
print(argmax):
tensor([[[[1, 1, 1, 2],
          [0, 2, 1, 0],
          [1, 0, 2, 2],
          [2, 1, 1, 1]]],

        [[[0, 0, 0, 2],
          [0, 0, 1, 1],
          [2, 2, 2, 2],
          [0, 1, 2, 0]]]])

Output:
If k is a tensor of shape (2, 3, 4, 4), by definition, torch.argmax with axis=1 should give you an output of shape (2, 4, 4). To understand why this happens, you have to understand what happens in lower dimensions first.
If I have a 2D (2, 2) tensor A, like:
[[1,2],
 [3,4]]

Then torch.argmax(A, axis=1) gives the output of shape (2) with values (1, 1). The axis argument means axis along which to operate. So setting axis=1 means that it will look at values from each column one by one, before deciding a max. For row 0, it looks at column values 1, 2 and decides that 2 (at index 1) is the max. For row 1, it looks at column vales 3, 4 and decides that 4 (at index 1) is the max. So the argmax result is [1, 1].
Moving up to 3D, let's have a hypothetical array of dimensions (I, J, K). If we call argmax with axis = 1, we can break it down to the following:
I, J, K = 3, 4, 5
A = torch.rand(I, J, K)
out = torch.zeros((I, K), dtype=torch.int32)

for i in range(I):
    for k in range(K):
        out[i,k] = torch.argmax(A[i,:,k])

print(out)
print(torch.argmax(A, axis=1))

Out:
tensor([[3, 3, 2, 3, 2],
        [1, 1, 0, 1, 0],
        [0, 1, 0, 3, 3]], dtype=torch.int32)
tensor([[3, 3, 2, 3, 2],
        [1, 1, 0, 1, 0],
        [0, 1, 0, 3, 3]])

So what happens is, in your 3D tensor, you're once again calculating argmax along the columns/axis 1. So for each unique pair of (i, k), you have exactly J values along the axis 1, right? The index of the maximum value within those J values is inserted into position (i,k) of the output.
If you understand this, then you can understand what happens in 4D. For any 4D tensor of dimensions (I, J, K, L), if you call argmax with axis=1, then for each combination of (i, k, l) you'll have exactly J values along axis 1 - and the argmax of those J values will be present at output[i,k,l].
The keepdims argument is merely conserving the number of dimensions of your matrix. For example, argmax at axis 1 on the 4D matrix gives a 3D result of shape (I,K,L), but using keepdims, the result will be 4D as well with the shape (I,1,K,L).


Instruction: Extracting Intermediate layer outputs of a CNN in PyTorch

Input:
I am using a Resnet18 model.
ResNet(
  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (relu): ReLU(inplace=True)
  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
  (layer1): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
    (1): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer2): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer3): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (layer4): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )
  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))
  (fc): Linear(in_features=512, out_features=1000, bias=True)
)

I want to extract the outputs only from layer2, layer3, layer4 & I don't want the avgpool and fc outputs.
How do I achieve this ?
class BasicBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1, padding=1) -> None:
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels,
                               3, stride, padding=padding, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(out_channels, out_channels,
                               3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)
        if in_channels != out_channels:
            l1 = nn.Conv2d(in_channels, out_channels,
                           kernel_size=1, stride=stride, bias=False)
            l2 = nn.BatchNorm2d(out_channels)
            self.downsample = nn.Sequential(l1, l2)
        else:
            self.downsample = None

    def forward(self, xb):
        prev = xb
        x = self.relu(self.bn1(self.conv1(xb)))
        x = self.bn2(self.conv2(x))
        if self.downsample is not None:
            prev = self.downsample(xb)
        x = x + prev
        return self.relu(x)

class CustomResnet(nn.Module):
    def __init__(self, pretrained:bool=True) -> None:
        super(CustomResnet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7,stride=2, padding=3, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
        self.layer1 = nn.Sequential(BasicBlock( 64, 64, stride=1), BasicBlock(64, 64))
        self.layer2 = nn.Sequential(BasicBlock(64, 128, stride=2), BasicBlock(128, 128))
        self.layer3 = nn.Sequential(BasicBlock(128, 256, stride=2), BasicBlock(256, 256))
        self.layer4 = nn.Sequential(BasicBlock(256, 512, stride=2), BasicBlock(512, 512))

    def forward(self, xb):
        x = self.maxpool(self.relu(self.bn1(self.conv1(xb))))
        x = self.layer1(x)
        x2 = x = self.layer2(x)
        x3 = x = self.layer3(x)
        x4 = x = self.layer4(x)
        return [x2, x3, x4]

I guess one solution would be this .. But is there any other way without writing this while lot of code? Also is it possible to load in the pre-trained weights given by torchvision in the above modified ResNet model.

Output:
If you know how the forward method is implemented, then you can subclass the model, and override the forward method only.
If you are using the pre-trained weights of a model in PyTorch, then you already have access to the code of the model. So, find where the code of the model is, import it, subclass the model, and override the forward method.
For example:

class MyResNet18(Resnet):

  def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)

  def forward(self, xb):
    x = self.maxpool(self.relu(self.bn1(self.conv1(xb))))
    x = self.layer1(x)
    x2 = x = self.layer2(x)
    x3 = x = self.layer3(x)
    x4 = x = self.layer4(x)
    return [x2, x3, x4]

and you are done.


Instruction: Getting model class labels from torchvision pretrained models

Input:
I am using a pre-trained Alexnet model (without fine-tuning) from torchvision. The issue is that even though I am able to run the model on some data and get the output probability distribution, I am unable to find class labels to map it to.
Following this official documentation
import torch
model = torch.hub.load('pytorch/vision:v0.6.0', 'alexnet', pretrained=True)
model.eval()

AlexNet(
  (features): Sequential(
    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))
    (1): ReLU(inplace=True)
    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (4): ReLU(inplace=True)
    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (7): ReLU(inplace=True)
    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (9): ReLU(inplace=True)
    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace=True)
    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (avgpool): AdaptiveAvgPool2d(output_size=(6, 6))
  (classifier): Sequential(
    (0): Dropout(p=0.5, inplace=False)
    (1): Linear(in_features=9216, out_features=4096, bias=True)
    (2): ReLU(inplace=True)
    (3): Dropout(p=0.5, inplace=False)
    (4): Linear(in_features=4096, out_features=4096, bias=True)
    (5): ReLU(inplace=True)
    (6): Linear(in_features=4096, out_features=1000, bias=True)
  )
)

Following some steps on processing the image, I am able to use it to get an output for a single image as a (1,1000) dim vector which I will use a softmax on to get a probability distribution -
#Output -

tensor([-1.6531e+00, -4.3505e+00, -1.8172e+00, -4.2143e+00, -3.1914e+00,
         3.4163e-01,  1.0877e+00,  5.9350e+00,  8.0425e+00, -7.0242e-01,
        -9.4130e-01, -6.0822e-01, -2.4097e-01, -1.9946e+00, -1.5288e+00,
        -3.2656e+00, -5.5800e-01,  1.0524e+00,  1.9211e-01, -4.7202e+00,
        -3.3880e+00,  4.3048e+00, -1.0997e+00,  4.6132e+00, -5.7404e-03,
        -5.3437e+00, -4.7378e+00, -3.3974e+00, -4.1287e+00,  2.9064e-01,
        -3.2955e+00, -6.7051e+00, -4.7232e+00, -4.1778e+00, -2.1859e+00,
        -2.9469e+00,  3.0465e+00, -3.5882e+00, -6.3890e+00, -4.4203e+00,
        -3.3685e+00, -5.0983e+00, -4.9006e+00, -5.5235e+00, -3.7233e+00,
        -4.0204e+00,  2.6998e-01, -4.4702e+00, -5.6617e+00, -5.4880e+00,
        -2.6801e+00, -3.2129e+00, -1.6294e+00, -5.2289e+00, -2.7495e+00,
        -2.6286e+00, -1.8206e+00, -2.3196e+00, -5.2806e+00, -3.7652e+00,
        -3.0987e+00, -4.1421e+00, -5.2531e+00, -4.6505e+00, -3.5815e+00,
        -4.0189e+00, -4.0008e+00, -4.5512e+00, -3.2248e+00, -7.7903e+00,
        -1.4484e+00, -3.8347e+00, -4.5611e+00, -4.3681e+00,  2.7234e-01,
        -4.0162e+00, -4.2136e+00, -5.4524e+00,  1.1744e+00, -4.7785e+00,
        -1.8335e+00,  4.1288e-01,  2.2239e+00, -9.9919e-02,  4.8216e+00,
        -8.4304e-01,  5.6911e-01, -4.0484e+00, -3.3013e+00,  2.8698e+00,
        -1.1419e+00, -9.1690e-01, -2.9284e+00, -2.6097e+00, -1.8213e-01,
        -2.5429e+00, -2.1095e+00,  2.2419e+00, -1.6280e+00,  7.4458e+00,
         2.3184e+00, -5.7408e+00, -7.4332e-01, -5.4066e+00,  1.5177e+01,
        -4.4737e-02,  1.8237e+00, -3.7741e+00,  9.2271e-01, -4.3687e-01,
        -1.4003e+00, -4.3026e+00,  6.3782e-01, -1.0808e+00, -1.4173e+00,
         2.6194e+00, -3.8418e+00,  1.1598e+00, -2.6876e+00, -3.6103e+00,
        -4.9281e+00, -4.1411e+00, -3.3603e+00, -3.4296e+00, -1.4997e+00,
        -2.8381e+00, -1.2843e+00,  1.5745e+00, -1.7449e+00,  4.2903e-01,
         3.1234e-01, -2.8206e+00,  3.6688e-01, -2.1033e+00,  1.6481e+00,
         1.4222e+00, -2.7303e+00, -3.6292e+00,  1.2864e+00, -2.5541e+00,
        -2.9663e+00, -4.1575e+00, -3.1954e+00, -4.6487e-01,  1.8916e+00,
        -7.4721e-01,  4.5986e+00, -2.5443e+00, -6.2003e+00, -1.3215e+00,
        -2.6225e+00,  9.9639e+00,  9.7772e+00,  9.6715e+00,  9.0857e+00,...

Where do I get the class labels from? I couldn't find any method that let me get that from the model object.

Output:
You cannot, unfortunately, get class label names directly from the torchvision models. However, these models are trained on the ImageNet dataset (hence the 1000 classes).
You have to get the class name mapping off the web as far as I know; there's no way to get it off torch. Previously, you could download ImageNet directly using torchvision.datasets.ImageNet, which had a built-in label to class name converter. Now the download link isn't publicly available and requires a manual download, before it can be used by datasets.ImageNet.
So you can simply search for the class to label mapping of ImageNet online, rather than downloading the data or attempting with torch. Try here for example.


Instruction: Simultaneous reads of the same PyTorch torchvision.datasets object

Input:
Consider the following piece of code to fetch a data set for training from torchvision.datasets and to create a DataLoader for it.
import torch
from torchvision import datasets, transforms

training_set_mnist = datasets.MNIST('./mnist_data', train=True, download=True)
train_loader_mnist = torch.utils.data.DataLoader(training_set_mnist, batch_size=128,
                                                 shuffle=True)

Assume that several Python processes have access to the folder ./mnist_data and execute the above piece of code simultaneously; in my case, each process is a different machine on a cluster and the data set is stored in an NFS location accessible by everyone. You may also assume that the data is already downloaded in this folder so download=True should have no effect. Moreover, each process may use a different seed, as set by torch.manual_seed().
I would like to know whether this scenario is allowed in PyTorch. My main concern is whether the above code can change the data folders or files in ./mnist_data such that if ran by multiple processes it can potentially lead to unexpected behavior or other issues. Also, given that shuffle=True I would expect that if 2 or more processes try to create the DataLoader each of them will get a different shuffling of the data assuming that the seeds are different. Is this true?

Output:
My main concern is whether the above code can change the data folders
or files in ./mnist_data such that if ran by multiple processes it can
potentially lead to unexpected behavior or other issues.

You will be fine as processes are only reading data, not modifying in (loading tensors with data into RAM in case of MNIST). Please notice processes do not share memory addresses, hence tensor with data will be loaded multiple times (which shouldn't be a big problem in case of MNIST).

Also, given that shuffle=True I would expect that if 2 or more
processes try to create the DataLoader each of them will get a
different shuffling of the data assuming that the seeds are different.

shuffle=True has nothing to do with data itself. What it does, is it get __len__() of provided dataset, makes a range [0, __len__()) and this range is shuffled and used to index dataset's __getitem__. Check out this section for more info about Samplers.


Instruction: Pytorch: The size of tensor a (24) must match the size of tensor b (48) at non-singleton dimension 3

Input:
Below code works fine and generate proper results.
import torch
import torch.nn as nn
import torch.nn.functional as F

from modules import ConvLSTMCell, Sign

class EncoderCell(nn.Module):
    def __init__(self):
        super(EncoderCell, self).__init__()

        self.conv = nn.Conv2d(
            3, 64, kernel_size=3, stride=2, padding=1, bias=False)
        self.rnn1 = ConvLSTMCell(
            64,
            256,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            256,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)

    def forward(self, input, hidden1, hidden2, hidden3):
        x = self.conv(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]

        return x, hidden1, hidden2, hidden3

class Binarizer(nn.Module):
    def __init__(self):
        super(Binarizer, self).__init__()
        self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
        self.sign = Sign()

    def forward(self, input):
        feat = self.conv(input)
        x = F.tanh(feat)
        return self.sign(x)

class DecoderCell(nn.Module):
    def __init__(self):
        super(DecoderCell, self).__init__()

        self.conv1 = nn.Conv2d(
            32, 512, kernel_size=1, stride=1, padding=0, bias=False)
        self.rnn1 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            128,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            128,
            256,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.rnn4 = ConvLSTMCell(
            64,
            128,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.conv2 = nn.Conv2d(
            32, 3, kernel_size=1, stride=1, padding=0, bias=False)

    def forward(self, input, hidden1, hidden2, hidden3, hidden4):
        x = self.conv1(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]
        x = F.pixel_shuffle(x, 2)

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]
        x = F.pixel_shuffle(x, 2)

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]
        x = F.pixel_shuffle(x, 2)

        hidden4 = self.rnn4(x, hidden4)
        x = hidden4[0]
        x = F.pixel_shuffle(x, 2)

        x = F.tanh(self.conv2(x)) / 2
        return x, hidden1, hidden2, hidden3, hidden4

Now i have changed in self.con and add pretrained resent with layer. Now it shows tensor mismatched error after training. All things are same just add this line in code. I put ** in those line
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models

from modules import ConvLSTMCell, Sign

class EncoderCell(nn.Module):
    def __init__(self):
        super(EncoderCell, self).__init__()

        #self.conv = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)

        **resConv = models.resnet50(pretrained=True)
        resConv.layer4 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
        self.conv = resConv.layer4**

        self.rnn1 = ConvLSTMCell(
            64,
            256,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            256,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=2,
            padding=1,
            hidden_kernel_size=1,
            bias=False)

    def forward(self, input, hidden1, hidden2, hidden3):

        x = self.conv(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]

        return x, hidden1, hidden2, hidden3

class Binarizer(nn.Module):
    def __init__(self):
        super(Binarizer, self).__init__()
        self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
        self.sign = Sign()

    def forward(self, input):
        feat = self.conv(input)
        x = F.tanh(feat)
        return self.sign(x)

class DecoderCell(nn.Module):
    def __init__(self):
        super(DecoderCell, self).__init__()

        **resConv = models.resnet50(pretrained=True)
        resConv.layer4 = nn.Conv2d(32, 512, kernel_size=3, stride=2, padding=1, bias=False)
        self.conv1 = resConv.layer4**

        self.rnn1 = ConvLSTMCell(
            512,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn2 = ConvLSTMCell(
            128,
            512,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=1,
            bias=False)
        self.rnn3 = ConvLSTMCell(
            128,
            256,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)
        self.rnn4 = ConvLSTMCell(
            64,
            128,
            kernel_size=3,
            stride=1,
            padding=1,
            hidden_kernel_size=3,
            bias=False)

        **resConv2 = models.resnet50(pretrained=True)
        resConv2.layer4 = nn.Conv2d(32, 3, kernel_size=1, stride=1, padding=0, bias=False)
        self.conv2 = resConv2.layer4**

    def forward(self, input, hidden1, hidden2, hidden3, hidden4):
        x = self.conv1(input)

        hidden1 = self.rnn1(x, hidden1)
        x = hidden1[0]
        x = F.pixel_shuffle(x, 2)

        hidden2 = self.rnn2(x, hidden2)
        x = hidden2[0]
        x = F.pixel_shuffle(x, 2)

        hidden3 = self.rnn3(x, hidden3)
        x = hidden3[0]
        x = F.pixel_shuffle(x, 2)

        hidden4 = self.rnn4(x, hidden4)
        x = hidden4[0]
        x = F.pixel_shuffle(x, 2)

        x = F.tanh(self.conv2(x)) / 2
        return x, hidden1, hidden2, hidden3, hidden4

Output:
You are doing it a wrong way, some explanation is,
    **resConv = models.resnet50(pretrained=True) # you are reading a model

now you are replacing the layer in that model with newly initialized layer. Secondly, layer4 in resnet50 is a sequential block containing multiple layers. Use print to see exact the layers in model.
    resConv.layer4 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)

here you are using new layer.
self.conv = resConv.layer4**

As per your query regarding usage of pretrained layer, you should do it like this,
resConv = models.resnet50(pretrained=True)
print(resConv) #see the layer which you want to use
self.conv = resConv.conv1 # replace conv1 with that layer
# note: conv1 is the name of first conv layer in resnet

To add to this, I would also recommend acquiring and adding this layer (or the weights and biases) outside of the object initialization. Something like:
enc = EncoderCell()
resnet50 = models.resnet50(pretrained=True)

and then either
enc.conv = resnet50.conv1

or more ideally
enc.conv.load_state_dict(resnet50.layer1.state_dict())

The reason being, calling state_dict() on a nn.Module class creates a clone of the parameters (weights and biases in this case) which can be loaded via nn.Module.load_state_dict() method as long as the two instances of nn.Module share the same shape. So you get the pretrained weights and they are completely detached from the pretrained model. Then you can get rid of the pretrained model since it could be rather large in memory.
del resnet50


Instruction: RuntimeError: cuda runtime error (30) : unknown error at ..\aten\src\THC\THCGeneral.cpp:87

Input:
I have tried installing CUDA on Windows 10 to train nerual networks in my GPU(NVIDIA GeForce 710) but I get the following error when I tried to load an initial model.
Here's the code I am running:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

device
"""# 1. Create the classifier"""

C = nn.Sequential(Flatten(), nn.Linear(784,200), nn.ReLU(),
                            nn.Linear(200,100), nn.ReLU(),
                            nn.Linear(100,100), nn.ReLU(),
                            nn.Linear(100,10))

"""Upload the trained model"""

C.load_state_dict(torch.load("C.pt",map_location='cuda'))

And this is the error I am getting:
C.load_state_dict(torch.load("C.pt",map_location='cuda'))
Traceback (most recent call last):

  File "<ipython-input-2-358f76f483ed>", line 1, in <module>
    C.load_state_dict(torch.load("C.pt",map_location='cuda'))

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 368, in load
    return _load(f, map_location, pickle_module)

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 542, in _load
    result = unpickler.load()

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 505, in persistent_load
    data_type(size), location)

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 385, in restore_location
    return default_restore_location(storage, map_location)

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 114, in default_restore_location
    result = fn(storage, location)

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\serialization.py", line 96, in _cuda_deserialize
    return obj.cuda(device)

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\_utils.py", line 68, in _cuda
    with torch.cuda.device(device):

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\cuda\__init__.py", line 229, in __enter__
    _lazy_init()

  File "C:\Users\usuario\anaconda3\lib\site-packages\torch\cuda\__init__.py", line 162, in _lazy_init
    torch._C._cuda_init()

RuntimeError: cuda runtime error (30) : unknown error at ..\aten\src\THC\THCGeneral.cpp:87

I already installed cudNN and these are the versions I am using.
Python 3.7.6
CUDA 8.0
Pytorch 1.0.1

Output:
Restarting your computer might fix the issue
reference
if it doesn't make sure that you
Re-install latest GPU driver
Reboot
Ensure you have admin access


Instruction: Computing matrix derivatives with torch.autograd.grad (PyTorch)

Input:
I am trying to compute matrix derivatives in PyTorch using torch.autograd.grad however I am running into few issues. Here is a minimal working example to reproduce the error.
theta = torch.tensor(np.random.uniform(low=-np.pi, high=np.pi), requires_grad=True)
rot_mat = torch.tensor([[torch.cos(theta), torch.sin(theta), 0],
                        [-torch.sin(theta), torch.cos(theta), 0]],
                        dtype=torch.float, requires_grad=True)
torch.autograd.grad(outputs=rot_mat,
                    inputs=theta, grad_outputs=torch.ones_like(rot_mat),
                    create_graph=True, retain_graph=True)

This code results in the error "One of the differentiated Tensors appears to not have been used in the graph. Set allow_unused=True if this is the desired behavior."
I tried using allow_unused=True but the gradients are returned as None. I am not sure what is causing the graph to be disconnected here.

Output:
Pytorch autograd graph will be created only if pytorch functions are used.
I think python 2d list used while creating rot_mat disconnects the graph. So using torch functions create rotation matrix and also just use backward() function to compute gradients. Here's sample code:
import torch
import numpy as np

theta   = torch.tensor(np.random.uniform(low=-np.pi, high=np.pi), requires_grad=True)

# create required values and convert it to torch 1d tensor
cos_t   = torch.cos(theta).view(1)
sin_t   = torch.sin(theta).view(1)
msin_t  = -sin_t
zero    = torch.zeros(1)

# create rotation matrix using only pytorch functions
rot_1d  = torch.cat((cos_t, sin_t, zero, msin_t, cos_t, zero))
rot_mat = rot_1d.view((2, 3))

# Autograd
rot_mat.backward(torch.ones_like(rot_mat))

# gradient
print(theta.grad)


Instruction: Pytorch CUDA OutOfMemory Error while training

Input:
I'm trying to train a PyTorch FLAIR model in AWS Sagemaker.
While doing so getting the following error:
RuntimeError: CUDA out of memory. Tried to allocate 84.00 MiB (GPU 0; 11.17 GiB total capacity; 9.29 GiB already allocated; 7.31 MiB free; 10.80 GiB reserved in total by PyTorch)

For training I used sagemaker.pytorch.estimator.PyTorch class.
I tried with different variants of instance types from ml.m5, g4dn to p3(even with a 96GB memory one).
In the ml.m5 getting the error with CPUmemoryIssue, in g4dn with GPUMemoryIssue and in the P3 getting GPUMemoryIssue mostly because Pytorch is using only one of the GPU of 12GB out of 8*12GB.
Not getting anywhere to complete this training, even in local tried with a CPU machine and got the following error:
RuntimeError: [enforce fail at ..\c10\core\CPUAllocator.cpp:72] data. DefaultCPUAllocator: not enough memory: you tried to allocate 67108864 bytes. Buy new RAM!

The model training script:
    corpus = ClassificationCorpus(data_folder, test_file='../data/exports/val.csv', train_file='../data/exports/train.csv')

    print("finished loading corpus")

    word_embeddings = [WordEmbeddings('glove'), FlairEmbeddings('news-forward-fast'), FlairEmbeddings('news-backward-fast')]

    document_embeddings = DocumentLSTMEmbeddings(word_embeddings, hidden_size=512, reproject_words=True, reproject_words_dimension=256)

    classifier = TextClassifier(document_embeddings, label_dictionary=corpus.make_label_dictionary(), multi_label=False)

    trainer = ModelTrainer(classifier, corpus, optimizer=Adam)

    trainer.train('../model_files', max_epochs=12,learning_rate=0.0001, train_with_dev=False, embeddings_storage_mode="none")

P.S.: I was able to train the same architecture with a smaller dataset in my local GPU machine with a 4GB GTX 1650 DDR5 memory and it was really quick.

Output:
Okay, so after 2 days of continuous debugging was able to find out the root cause.
What I understood is Flair does not have any limitation on the sentence length, in the sense the word count, it is taking the highest length sentence as the maximum.
So there it was causing issue, as in my case there were few content with 1.5 lakh rows which is too much to load the embedding of into the memory, even a 16GB GPU.
So there it was breaking.
To solve this: For content with this much lengthy words, you can take chunk of n words(10K in my case) from these kind of content from any portion(left/right/middle anywhere) and trunk the rest, or simply ignore those records for training if it is very minimal in comparative count.
After this I hope you will be able to progress with your training, as it happened in my case.
P.S.: If you are following this thread and face similar issue feel free to comment back so that I can explore and help on your case of the issue.


Instruction: How to train Pytorch CNN with two or more inputs

Input:
I have a big image, multiple events in the image can impact the classification. I am thinking to split big image into small chunks and get features from each chunk and concatenate outputs together for prediction.
My code is like:
train_load_1 = DataLoader(dataset=train_dataset_1, batch_size=100, shuffle=False)
train_load_2 = DataLoader(dataset=train_dataset_2, batch_size=100, shuffle=False)
train_load_3 = DataLoader(dataset=train_dataset_3, batch_size=100, shuffle=False)

test_load_1 = DataLoader(dataset=test_dataset_1, batch_size=100, shuffle=True)
test_load_2 = DataLoader(dataset=test_dataset_2, batch_size=100, shuffle=True)
test_load_3 = DataLoader(dataset=test_dataset_3, batch_size=100, shuffle=True)

class Net(nn.Module):
   def __init__(self):
      super(Net, self).__init__()
      self.conv = nn.Conv2d( ... )  # set up your layer here
      self.fc1 = nn.Linear( ... )  # set up first FC layer
      self.fc2 = nn.Linear( ... )  # set up the other FC layer

   def forward(self, x1, x2, x3):
      o1 = self.conv(x1)
      o2 = self.conv(x2)
      o3 = self.conv(x3)
      combined = torch.cat((o1.view(c.size(0), -1),
                            o2.view(c.size(0), -1),
                            o3.view(c.size(0), -1)), dim=1)
      out = self.fc1(combined)
      out = self.fc2(out)
      return F.softmax(x, dim=1)

model = Net().to(device)
optimizer = optim.SGD(model.parameters(), lr=0.01)

for epoch in epochs:
   model.train()

   for batch_idx, (inputs, labels) in enumerate(train_loader_1):
   **### I am stuck here, how to enumerate all three train_loader to pass input_1, input_2, input_3 into model and share the same label? Please note in train_loader I have set shuffle=False, this is to make sure train_loader_1, train_loader_2, train_loader_3 are getting the same label **

Thank you for your help!

Output:
Instead of using 3 separate dataLoader elements, you can use a single dataLoader element where each of the datapoint contains 3 separate parts of the image.
Like this:
dataLoader = [[[img1_part1],[img1_part2],[img1_part3], label1], [[img2_part1],[img2_part2],[img2_part3], label2]....]

This way you can use that in training loop as:
for img in dataLoader:
    part1,part2,part3,label = img
    out = model.forward(part1,part2,part3)
    loss = loss_fn(out, label)
    loss.backward()
    optimizer.step()


Instruction: can anyone explain what "out = self(images)" do in below code

Input:
I am not able to understand, if prediction is calculated in forward method, then why there is need "out = self(images)" and what it will do. I am bit confuse about this code.
class MnistModel(nn.Module):
def __init__(self):
    super().__init__()
    self.linear = nn.Linear(input_size, num_classes)

def forward(self, xb):
    xb = xb.reshape(-1, 784)
    out = self.linear(xb)
    return out

def training_step(self, batch):
    images, labels = batch
    out = self(images)                  # Generate predictions
    loss = F.cross_entropy(out, labels) # Calculate loss
    return loss

def validation_step(self, batch):
    images, labels = batch
    out = self(images)                    # Generate predictions
    loss = F.cross_entropy(out, labels)   # Calculate loss
    acc = accuracy(out, labels)           # Calculate accuracy
    return {'val_loss': loss, 'val_acc': acc}

def validation_epoch_end(self, outputs):
    batch_losses = [x['val_loss'] for x in outputs]
    epoch_loss = torch.stack(batch_losses).mean()   # Combine losses
    batch_accs = [x['val_acc'] for x in outputs]
    epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies
    return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}

def epoch_end(self, epoch, result):
    print("Epoch [{}], val_loss: {:.4f}, val_acc: {:.4f}".format(epoch, result['val_loss'], result['val_acc']))

model = MnistModel()

Output:
In Python, self refers to the instance that you have created from a class (similar to this in Java and C++). An instance is callable, which means it may be called like a function itself, if method __call__ have been overridden.
Example:
class A:
    def __init__(self):
        pass
    def __call__(self, x, y):
        return x + y

a = A()
print(a(3,4)) # Prints 7

In your case, __call__ method is implemented in super class nn.Module.
As it is a neural network module it needs an input placeholder. "out" is the placeholder for the data that is going to be forward the output of the module to the next layer or module of your model.
In the case of nn.Module class instances (and those that inherit from the class) the forward method is what is used as the __call__ method. At least where it is defined with respect to the nn.Module class.


Instruction: Input and hidden tensors are not at the same device, found input tensor at cuda:0 and hidden tensor at cpu

Input:
here is my code for lstm network, I instantiated it and passed to Cuda device but still getting the error that hidden and inputs are not in same device
class LSTM_net(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
    super(LSTM_net, self).__init__()
    self.hidden_size = hidden_size
    self.lstm_cell = nn.LSTM(input_size, hidden_size)
    self.h2o = nn.Linear(hidden_size, output_size)
    self.softmax = nn.LogSoftmax(dim=1)

def forward(self, input, hidden_0=None, hidden_1=None, hidden_2=None):
    input=resnet(input)
    input=input.unsqueeze(0)
    out_0, hidden_0 = self.lstm_cell(input, hidden_0)
    out_1, hidden_1 = self.lstm_cell(out_0+input, hidden_1)
    out_2, hidden_2 = self.lstm_cell(out_1+input, hidden_2)
    output = self.h2o(hidden_2[0].view(-1, self.hidden_size))
    output = self.softmax(output)
    return output,hidden_0,hidden_1, hidden_2

def init_hidden(self, batch_size = 1):
    return (torch.zeros(1, batch_size, self.hidden_size), torch.zeros(1, batch_size, self.hidden_size))

net1=LSTM_net(input_size=1000,hidden_size=1000, output_size=100)

net1=net1.to(device)

pic of connections that I want to make, plz guide me to implement it
click here for an image of error massege

Output:
Edit: I think I see the problem now. Try changing
    def init_hidden(self, batch_size = 1):
        return (torch.zeros(1, batch_size, self.hidden_size), torch.zeros(1, batch_size, self.hidden_size))

to
    def init_hidden(self, batch_size = 1):
        return (torch.zeros(1, batch_size, self.hidden_size).cuda(), torch.zeros(1, batch_size, self.hidden_size).cuda())

This is because each of the tensors created by init_hidden method are not data attributes in the parent object of the function. So they do not have cuda() applied to them when you apply cuda() to an instance of the model object.
Try calling .cuda() on all the tensors/variables and models involved.
net1.cuda() # net1.to(device) for device == cuda:0 works fine also
            # cuda() is more succinct, though
input.cuda()

# now, calling net1 on a tensor named input should not produce the error.
out = net1(input)


Instruction: Why do I get CUDA out of memory when running PyTorch model [with enough GPU memory]?

Input:
I am asking this question because I am successfully training a segmentation network on my GTX 2070 on laptop with 8GB VRAM and I use exactly the same code and exactly the same software libraries installed on my desktop PC with a GTX 1080TI and it still throws out of memory.
Why does this happen, considering that:

The same Windows 10 + CUDA 10.1 + CUDNN 7.6.5.32 + Nvidia Driver 418.96 (comes along with CUDA 10.1) are both on laptop and on PC.

The fact that training with TensorFlow 2.3 runs smoothly on the GPU on my PC, yet it fails allocating memory for training only with PyTorch.

PyTorch recognises the GPU (prints GTX 1080 TI) via the command : print(torch.cuda.get_device_name(0))

PyTorch allocates memory when running this command: torch.rand(20000, 20000).cuda() #allocated 1.5GB of VRAM.

What is the solution to this?

Output:
Most of the people (even in the thread below) jump to suggest that decreasing the batch_size will solve this problem. In fact, it does not in this case. For example, it would have been illogical for a network to train on 8GB VRAM and yet to fail to train on 11GB VRAM, considering that there were no other applications consuming video memory on the system with 11GB VRAM and the exact same configuration is installed and used.
The reason why this happened in my case was that, when using the DataLoader object, I set a very high (12) value for the workers parameter. Decreasing this value to 4 in my case solved the problem.
In fact, although at the bottom of the thread, the answer provided by Yurasyk at https://github.com/pytorch/pytorch/issues/16417#issuecomment-599137646 pointed me in the right direction.
Solution: Decrease the number of workers in the PyTorch DataLoader. Although I do not exactly understand why this solution works, I assume it is related to the threads spawned behind the scenes for data fetching; it may be the case that, on some processors, such an error appears.


Instruction: Optimizer for an RNN using pytorch

Input:
The pytorch RNN tutorial uses
for p in net.parameters():
    p.data.add_(p.grad.data, alpha = -learning_rate)

as optimizer. Does anyone know the difference between doing that or doing the classical optimizer.step(), once an optimizer has been defined explicitly? Is there some special consideration one has to take into when training RNNs in regards to the optimizer?

Output:
It looks like the example uses a simple gradient descent algorithm to update:

where J is cost.
If the optimizer your using is a simple gradient descent tool, then there is no difference between using optimizer.step() and the code in the example.
I know that's not a super exciting answer to your question, because it depends on how the step() function is written. Check out this page to learn about step() and this page to learn more about torch.optim.


Instruction: How do I fix the Dataset to return desired output (pytorch)

Input:
I am trying to use information from the outside functions to decide which data to return. Here, I have added a simplified code to demonstrate the problem. When I use num_workers = 0, I get the desired behavior (The output after 3 epochs is 18). But, when I increase the value of num_workers, the output after each epoch is the same. And the global variable remains unchanged.
from torch.utils.data import Dataset, DataLoader

x = 6
def getx():
    global x
    x+=1
    print("x: ", x)
    return x

class MyDataset(Dataset):
    def __init__(self):
        pass

    def __getitem__(self, index):
        global x
        x = getx()
        return x

    def __len__(self):
        return 3

dataset = MyDataset()
loader = DataLoader(
    dataset,
    num_workers=0,
    shuffle=False
)

for epoch in range(4):
    for idx, data in enumerate(loader):
        print('Epoch {}, idx {}, val: {}'.format(epoch, idx, data))

The final output when num_workers=0 is 18 as expected. But when num_workers>0, x remains unchanged (The final output is 6).
How can I get a similar behavior as num_workers=0 using num_workers>0(i.e.How to ensure the __getitem__ function of dataloader changes the global variable x's value )?

Output:
The reason for this is the underlying nature of multiprocessing in python. Setting num_workers means that your DataLoader creates that number of sub-processes. Each sub-process is effectively a separate python instance with its own global state, and has no idea of what's going on in the other processes.
A typical solution for this in python's multiprocessing is using a Manager. However, since your multiprocessing is being provided through the DataLoader, you have no way to work this in.
Fortunately, something else can be done. DataLoader actually relies on torch.multiprocessing, which in turn allows sharing of tensors between processes as long as they are in shared memory.
So what you can do is, simply use x as a shared tensor.
from torch.utils.data import Dataset, DataLoader
import torch

x = torch.tensor([6])
x.share_memory_()

def getx():
    global x
    x+=1
    print("x: ", x.item())
    return x

class MyDataset(Dataset):
    def __init__(self):
        pass

    def __getitem__(self, index):
        global x
        x = getx()
        return x

    def __len__(self):
        return 3

dataset = MyDataset()
loader = DataLoader(
    dataset,
    num_workers=2,
    shuffle=False
)

for epoch in range(4):
    for idx, data in enumerate(loader):
        print('Epoch {}, idx {}, val: {}'.format(epoch, idx, data))

Out:
x:  7
x:  8
x:  9
Epoch 0, idx 0, val: tensor([[7]])
Epoch 0, idx 1, val: tensor([[8]])
Epoch 0, idx 2, val: tensor([[9]])
x:  10
x:  11
x:  12
Epoch 1, idx 0, val: tensor([[10]])
Epoch 1, idx 1, val: tensor([[12]])
Epoch 1, idx 2, val: tensor([[12]])
x:  13
x:  14
x:  15
Epoch 2, idx 0, val: tensor([[13]])
Epoch 2, idx 1, val: tensor([[15]])
Epoch 2, idx 2, val: tensor([[14]])
x:  16
x:  17
x:  18
Epoch 3, idx 0, val: tensor([[16]])
Epoch 3, idx 1, val: tensor([[18]])
Epoch 3, idx 2, val: tensor([[17]])

While this works, it isn't perfect. Look at epoch 1, and notice that there are 2 12s rather than 11 and 12. This means that two separate processes have executed the line x+=1 before executing print. This is unavoidable as parallel processes are working on shared memory.
If you're familiar with operating system concepts, you may be able to further implement some sort of semaphore with an extra variable to control the access to x as needed - but as this goes beyond the scope of the question, I won't elaborate further.


Instruction: Finding the top k matches in Pytorch

Input:
I'm using the following code to find the topk matches using pytorch:
def find_top(self, x, y, n_neighbors, unit_vectors=False, cuda=False):
    if not unit_vectors:
        x = __to_unit_torch__(x, cuda=cuda)
        y = __to_unit_torch__(y, cuda=cuda)
    with torch.no_grad():
        d = 1. - torch.matmul(x, y.transpose(0, 1))
        values, indices = torch.topk(d, n_neighbors, dim=1, largest=False, sorted=True)
        return indices.cpu().numpy()

Unfortunately, it is throwing the following error:
values, indices = torch.topk(d, n_neighbors, dim=1, largest=False, sorted=True)
RuntimeError: invalid argument 5: k not in range for dimension at /pytorch/aten/src/THC/generic/THCTensorTopK.cu:23

The size of d is (1793,1)  . What am I missing?

Output: This error occurs when you call torch.topk with a k larger than the total number of classes. Reduce your argument and it should run fine.


Instruction: CNN trained model doesn't appear to be working

Input:
I've trained a CNN model and I would like to run the trained model against new data. However, it seems that the trained model isn't predicting the count correctly as it done during the training. I have a feeling that the model is not using the PTH file. Could someone please advise what I am doing wrong, please?
import argparse
import datetime
import glob
import os
import random
import shutil
import time
from os.path import join

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from torchvision.transforms import ToTensor
from tqdm import tqdm
import torch.optim as optim

from convnet3_eval import Convnet
from dataset2_eval import CellsDataset

parser = argparse.ArgumentParser('Predicting hits from pixels')
parser.add_argument('name',type=str,help='Name of experiment')
parser.add_argument('data_dir',type=str,help='Path to data directory containing images and gt.csv')
parser.add_argument('--weight_decay',type=float,default=0.0,help='Weight decay coefficient (something like 10^-5)')
parser.add_argument('--lr',type=float,default=0.0001,help='Learning rate')
args = parser.parse_args()

metadata = pd.read_csv(join(args.data_dir,'gt.csv'))
metadata.set_index('filename', inplace=True)

dataset = CellsDataset(args.data_dir,transform=ToTensor(),return_filenames=True)
dataset = DataLoader(dataset,num_workers=4,pin_memory=True)
model_path = '/base_model.pth'

model = Convnet()
optimizer = torch.optim.Adam(model.parameters(),lr=args.lr,weight_decay=args.weight_decay)

for images, paths in tqdm(dataset):

    targets = torch.tensor([metadata['count'][os.path.split(path)[-1]] for path in paths]) # B
    targets = targets.float()

    # code to print training data to a csv file
    filename=CellsDataset(args.data_dir,transform=ToTensor(),return_filenames=True)
    output = model(images) # B x 1 x 9 x 9 (analogous to a heatmap)
    preds = output.sum(dim=[1,2,3]) # predicted cell counts (vector of length B)
    print(preds)
    paths_test = np.array([paths])
    names_preds = np.hstack(paths)
    print(names_preds)
    df=pd.DataFrame({'Image_Name':names_preds, 'Target':targets.detach(), 'Prediction':preds.detach()})
    print(df)
    # save image name, targets, and predictions
    df.to_csv(r'model.csv', index=False, mode='a')

model.load_state_dict(torch.load(model_path))
model.eval()

Output:
Move the last two lines where you load the weights
model.load_state_dict(torch.load(model_path))
model.eval()

above the for loop right below where you initialize the model.


Instruction: RuntimeError: cudnn RNN backward can only be called in training mode

Input:
I have seen this problem the first time, I never encountered such an error in previous Python projects. Here is my training code:
def train(net, opt, criterion,ucf_train, batchsize,i):
    opt.zero_grad()
    total_loss = 0
    net=net.eval()
    net=net.train()
    for vid in range(i*batchsize,i*batchsize+batchsize,1):

        output=infer(net,ucf_train[vid])
        m=get_label_no(ucf_train[vid])
        m=m.cuda( )
        loss = criterion(output,m)
        loss.backward(retain_graph=True)
        total_loss += loss
        opt.step()       #updates wghts and biases

    return total_loss/n_points

code for infer(net,input)
def infer(net, name):
    net.eval()
    hidden_0 = net.init_hidden()
    hidden_1 = net.init_hidden()
    hidden_2 = net.init_hidden()
    video_path = fetch_ucf_video(name)
    cap = cv2.VideoCapture(video_path)
    resize=(224,224)
    T=FrameCapture(video_path)
    print(T)
    lim=T-(T%20)-2
    i=0
    while(1):
      ret, frame2 = cap.read()
      frame2= cv2.resize(frame2, resize)
    #  print(type(frame2))
      if (i%20==0 and i<lim):
          input=normalize(frame2)
          input=input.cuda()
          output,hidden_0,hidden_1, hidden_2  = net(input, hidden_0, hidden_1, hidden_2)
      elif (i>=lim):
          break
      i=i+1
    op=output
    torch.cuda.empty_cache()
    op=op.cuda()
    return op

I am getting this error, I tried with model.train() following this where net is my model:
 RuntimeError                              Traceback (most recent call last)
<ipython-input-62-42238f3f6877> in <module>()
----> 1 train(net1,opt,criterion,ucf_train,1,0)

2 frames
/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
    125     Variable._execution_engine.run_backward(
    126         tensors, grad_tensors, retain_graph, create_graph,
--> 127         allow_unreachable=True)  # allow_unreachable flag
    128
    129

RuntimeError: cudnn RNN backward can only be called in training mode

Output:
You should remove the net.eval() call that comes right after the def infer(net, name):
It needs to be removed because you call this infer function inside your training code. Your model needs to be in train mode throughout the the whole training.
And you never set your model back to train after calling eval as well, so that is the root of the exception you are getting. If you want to use this infer code in your test cases, you can cover that case with an if.
Also the net.eval() that comes right after the total_loss=0 assignment is not useful since you call net.train() right after that. You can also remove that one since it gets neutralized right in next line.
The updated code
def train(net, opt, criterion,ucf_train, batchsize,i):
    opt.zero_grad()
    total_loss = 0
    net=net.train()
    for vid in range(i*batchsize,i*batchsize+batchsize,1):
        output=infer(net,ucf_train[vid])
        m=get_label_no(ucf_train[vid])
        m=m.cuda( )
        loss = criterion(output,m)
        loss.backward(retain_graph=True)
        total_loss += loss
        opt.step()       #updates wghts and biases

    return total_loss/n_points

code for infer(net,input)
def infer(net, name, is_train=True):
    if not is_train:
        net.eval()
    hidden_0 = net.init_hidden()
    hidden_1 = net.init_hidden()
    hidden_2 = net.init_hidden()
    video_path = fetch_ucf_video(name)
    cap = cv2.VideoCapture(video_path)
    resize=(224,224)
    T=FrameCapture(video_path)
    print(T)
    lim=T-(T%20)-2
    i=0
    while(1):
      ret, frame2 = cap.read()
      frame2= cv2.resize(frame2, resize)
      #  print(type(frame2))
      if (i%20==0 and i<lim):
          input=normalize(frame2)
          input=input.cuda()
          output,hidden_0,hidden_1, hidden_2  = net(input, hidden_0, hidden_1, hidden_2)
      elif (i>=lim):
          break
      i=i+1
    op=output
    torch.cuda.empty_cache()
    op=op.cuda()
    return op


Instruction: Why does `is_same` in the C++ PyTorch API fail when comparing with the same tensor that is read from a file?

Input:
Why does torch::Tensor::is_same fail the following assertion? A tensor is written to a file using the C++ PyTorch API, then read again into another tensor, and is_same compares both tensors:
torch::Tensor x_sequence = torch::linspace(0, M_PI, 1000);
torch::save(x_sequence, "x_sequence.dat");
torch::Tensor x_read;
torch::load(x_read, "x_sequence.dat");
assert(x_read.is_same(x_sequence));

This results in:
int main(int, char**): Assertion `x_read.is_same(x_sequence)' failed.

using

python-pytorch, version 1.6.0-2 on Arch Linux
g++ (GCC) 10.1.0

Output:
torch::Tensor::is_same(const torch::Tensor& other)is defined here. It is important to notice that a Tensor is actually a pointer on an underlying TensorImpl class (which actually holds the data).
Thus, when you call is_same, what is checked is actually whether or not your pointers are the same, i.e whether your 2 tensors are pointing to the same underlying memory. Here is a very simple example to understand it well :
auto x = torch::randn({4,4});
auto copy = x;
auto clone = x.clone();
std::cout << x.is_same(copy) << " " << x.is_same(clone) << std::endl;
>>> 0 1

Here, the call to clone forces pytorch to copy the data in another memory location. Consequently, the pointers are different and is_same returns false.
If you want to actually compare the values, you have no choice but to compute the difference between the two tensors and compute how close to 0 this difference is.


Instruction: In a python iterator select other iterator based on conditions

Input:
In python I have an iterator returning an infinite string of indices in a fixed range [0, N] called Sampler. Actually I have a list of those and all they do is return indices in the range [0, N_0], [N_0, N_1], ..., [N_{n-1}, N_n].
What I now want to do is first select one of these iterators based on the length of their range, so I have a weights list [N_0, N_1 - N_0, ...] and I select one of these with:
    iterator_idx = random.choices(range(len(weights)), weights=weights/weights.sum())[0]

Next, what I want to do is create an iterator which randomly selects one of the iterators and selects a batch of M samples.
class BatchSampler:
    def __init__(self, M):
        self.M = M
        self.weights = [weight_list]

        self.samplers = [list_of_iterators]
        ]
        self._batch_samplers = [
            self.batch_sampler(sampler) for sampler in self.samplers
        ]

    def batch_sampler(self, sampler):
        batch = []
        for batch_idx in sampler:
            batch.append(batch_idx)
            if len(batch) == self.M:
                yield batch

        if len(batch) > 0:
            yield batch

    def __iter__(self):
        # First select one of the datasets.
        iterator_idx = random.choices(
            range(len(self.weights)), weights=self.weights / self.weights.sum()
        )[0]
        return self._batch_samplers[iterator_idx]

The issue with this is that iter() only seems to be called once, so only the first time iterator_idx is selected. Obviously this is wrong... What is the way around this?
This is a possible case when you would have multiple datasets in pytorch, but you want to sample only batches from one of the datasets.

Output:
Seems to me that you want to define your own container type.
I'll try to provide examples of a few standard ways to do so
(hopefully without missing too many details);
you should be able to reuse one of these simple examples,
into your own class.

Using just __getitem__ (support indexing & looping):
object.__getitem__

Called to implement evaluation of self[key].

class MyContainer:
  def __init__(self, sequence):
    self.elements = sequence  # Just something to work with.

  def __getitem__(self, key):
    # If we're delegating to sequences like built-in list,
    # invalid indices are handled automatically by them
    # (throwing IndexError, as per the documentation).
    return self.elements[key]

t = (1, 2, 'a', 'b')
c = MyContainer(t)
elems = [e for e in c]
assert elems == [1, 2, 'a', 'b']
assert c[1:-1] == t[1:-1] == (2, 'a')

Using the iterator protocol:
object.__iter__

object.__iter__(self)
This method is called when an iterator is required for a container. This method should return a new iterator object that can iterate over all the objects in the container. For mappings, it should iterate over the keys of the container.
Iterator objects also need to implement this method; they are required to return themselves. For more information on iterator objects, see Iterator Types.

Iterator Types

container.__iter__()
Return an iterator object. The object is required to support the iterator protocol described below.

The iterator objects themselves are required to support the following two methods, which together form the iterator protocol:
iterator.__iter__()
Return the iterator object itself. This is required to allow both containers and iterators to be used with the for and in statements.
iterator.__next__()
Return the next item from the container. If there are no further items, raise the StopIteration exception.

Once an iterator's __next__() method raises StopIteration, it must continue to do so on subsequent calls.

class MyContainer:
  class Iter:
    def __init__(self, container):
      self.cont = container
      self.pos = 0
      self.len = len(container.elements)

    def __iter__(self): return self
    def __next__(self):
      if self.pos == self.len: raise StopIteration
      curElem = self.cont.elements[self.pos]
      self.pos += 1
      return curElem

  def __init__(self, sequence):
    self.elements = sequence  # Just something to work with.

  def __iter__(self):
    return MyContainer.Iter(self)

t = (1, 2, 'a', 'b')
c = MyContainer(t)
elems = [e for e in c]
assert elems == [1, 2, 'a', 'b']

Using a generator:
Generator Types

Python's generators provide a convenient way to implement the iterator protocol. If a container object's iter() method is implemented as a generator, it will automatically return an iterator object (technically, a generator object) supplying the iter() and next() methods.

generator

A function which returns a generator iterator. It looks like a normal function except that it contains yield expressions for producing a series of values usable in a for-loop or that can be retrieved one at a time with the next() function.
Usually refers to a generator function, but may refer to a generator iterator in some contexts.

generator iterator

An object created by a generator function.

6.2.9. Yield expressions

Using a yield expression in a function's body causes that function to be a generator

class MyContainer:
  def __init__(self, sequence):
    self.elements = sequence  # Just something to work with.

  def __iter__(self):
    for e in self.elements: yield e

t = (1, 2, 'a', 'b')
c = MyContainer(t)
elems = [e for e in c]
assert elems == [1, 2, 'a', 'b']


Instruction: Correct Way to Fine-Tune/Train HuggingFace's Model from scratch (PyTorch)

Input:
For example, I want to train a BERT model from scratch but using the existing configuration. Is the following code the correct way to do so?
model = BertModel.from_pretrained('bert-base-cased')
model.init_weights()

Because I think the init_weights method will re-initialize all the weights.
Second question, if I want to change a bit the configuration, such as the number of hidden layers.
model = BertModel.from_pretrained('bert-base-cased', num_hidden_layers=10)
model.init_weights()

I wonder if the above is the correct way to do so. Because they don't appear to have an error when I run the above code.

Output:
In this way, you would unnecessarily download and load the pre-trained model weights. You can avoid that by downloading the BERT config
config = transformers.AutoConfig.from_pretrained("bert-base-cased")
model = transformers.AutoModel.from_config(config)

Both yours and this solution assume you want to tokenize the input in the same as the original BERT and use the same vocabulary. If you want to use a different vocabulary, you can change in the config before instantiating the model:
config.vocab_size = 123456

Similarly, you can change any hyperparameter that you want to have different from the original BERT.


Instruction: Missing XLA configuration when running pytorch/xla

Input:
I am trying to run GCP TPU with Pytorch/XLA, I am using a VM with debian-9-torch-xla-v20200818 image, I initiate the TPU and check it is running using ctpu status which shows that both the CPU and TPU are running, I then activate the torch-xla-nightly environment, but when I try to invoke this simple code:
import torch
import torch_xla
import torch_xla.core.xla_model as xm

dev = xm.xla_device()
t1 = torch.ones(3, 3, device = dev)
print(t1)

this error comes up:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/core/xla_model.py", line 231, in xla_device
devkind=devkind if devkind is not None else None)
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/core/xla_model.py", line 136, in get_xla_supported_devices
 xla_devices = _DEVICES.value
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/utils/utils.py", line 32, in value
self._value = self._gen_fn()
File "/anaconda3/envs/torch-xla-nightly/lib/python3.6/site-packages/torch_xla/core/xla_model.py", line 18, in <lambda>
_DEVICES = xu.LazyProperty(lambda: torch_xla._XLAC._xla_get_devices())
RuntimeError: tensorflow/compiler/xla/xla_client/computation_client.cc:274 : Missing XLA configuration

I tried everything but nothing seem to work.

Output: Take a look at this link as it seems to pertain to the issue. Maybe you didn't setup the XRT_TPU_CONFIG: (vm)$ export XRT_TPU_CONFIG="tpu_worker;0;$TPU_IP_ADDRESS:8470" Follow the instructions here and you should be fine.


Instruction: Iterator doesn't work with DataLoader on GPU

Input:
I'm using PyTorch on Google Colab, I'm getting this error when Using GPU,
TypeError                                 Traceback (most recent call last)
<ipython-input-33-41cdbc758ecd> in <module>()
----> 1 dataiter= iter(trainloader)

TypeError: '_SingleProcessDataLoaderIter' object is not callable

but wen using normal CPU there is no Error.
My code:
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import torch
import numpy as np
from torchvision import datasets, transforms

from collections import  OrderedDict

from torch import nn
from torch import  optim
import torch.nn.functional as F
import helper

transform = transforms.Compose([transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset= datasets.MNIST("MINIST_data/", download= True, train=True, transform=transform)
trainloader= torch.utils.data.DataLoader(trainset, batch_size= 64, shuffle=True)
dataiter= iter(trainloader)

Using enumerate instead of iter works with GPU but I don't know why, can someone explain the error to me and why it is happening !?

Output: You don't have to use iter. trainloader is already iterable. The loop should be done like this for data in trainloader: or for index, data in enumerate(trainloader):


Instruction: PyTorch method for returning element with highest count in a 1D tensor?

Input:
Is there a built-in PyTorch method that takes a 1D tensor, and returns the element in the tensor with the highest count?
For example, if we input torch.tensor([2,2,2,3,4,5]), the method should return 2 as it occurs the most. In case of a tie in frequency, the element with the lower value should be returned; inputting torch.tensor([1,1,2,2,4,5]) should return 1.
Just to be clear, I only wish to know if there's an existing built-in PyTorch method that does exactly this. If there's no such method, please refrain from posting the solution, as I'd like to try solving it on my own.

Output:
yes torch.mode() is builtin function(read here) which handles both of your conditions.
torch.mode(alpha,0) #alpha being the name of tensor


Instruction: Denoising linear autoencoder learns to output a constant instead of denoising

Input:
I am trying to create a denoising autoencoder for 1d cyclic signals like cos(x) etc.
The process of creating the dataset is that I pass a list of cyclic functions and for each example generated it rolls random coefficients for each function in the list so every function generated is different yet cyclic. eg - 0.856cos(x) - 1.3cos(0.1x)
Then I add noise and normalize the signal to be between [0, 1).
Next, I train my autoencoder on it but it learns to output a constant (usually 0.5). my guess is that it happens because 0.5 is the usual mean value of the normalized functions. But this is not the result im aspiring to get at all.
I am providing the code I wrote for the autoencoder, the data generator and the training loop as well as two pictures depicting the problem im having.
first example:

second example:

Linear autoencoder:
class LinAutoencoder(nn.Module):
    def __init__(self, in_channels, K, B, z_dim, out_channels):
        super(LinAutoencoder, self).__init__()

        self.in_channels = in_channels
        self.K = K # number of samples per 2pi interval
        self.B = B # how many intervals
        self.out_channels = out_channels

        encoder_layers = []
        decoder_layers = []

        encoder_layers += [
            nn.Linear(in_channels * K * B, 2*z_dim, bias=True),
            nn.ReLU(),
            nn.Linear(2*z_dim, z_dim, bias=True),
            nn.ReLU(),
            nn.Linear(z_dim, z_dim, bias=True),
            nn.ReLU()
        ]

        decoder_layers += [
            nn.Linear(z_dim, z_dim, bias=True),
            nn.ReLU(),
            nn.Linear(z_dim, 2*z_dim, bias=True),
            nn.ReLU(),
            nn.Linear(2*z_dim, out_channels * K * B, bias=True),
            nn.Tanh()
        ]

        self.encoder = nn.Sequential(*encoder_layers)
        self.decoder = nn.Sequential(*decoder_layers)

    def forward(self, x):
        batch_size = x.shape[0]
        x_flat = torch.flatten(x, start_dim=1)
        enc = self.encoder(x_flat)
        dec = self.decoder(enc)
        res = dec.view((batch_size, self.out_channels, self.K * self.B))
        return res

The data generator:
def lincomb_generate_data(batch_size, intervals, sample_length, functions, noise_type="gaussian", **kwargs)->torch.tensor:

    channels = 1
    mul_term = 2 * np.pi / sample_length

    positions = np.arange(0, sample_length * intervals)
    x_axis = positions * mul_term
    X = np.tile(x_axis, (channels, 1))
    y = X

    Y = np.repeat(y[np.newaxis, :], batch_size, axis=0)

    if noise_type == "gaussian":
        # defaults to 0, 0.4
        noise_mean = kwargs.get("noise_mean", 0)
        noise_std  = kwargs.get("noise_std", 0.4)

        noise = np.random.normal(noise_mean, noise_std, Y.shape)

    if noise_type == "uniform":
        # defaults to 0, 1
        noise_low  = kwargs.get("noise_low", 0)
        noise_high = kwargs.get("noise_high", 1)

        noise = np.random.uniform(noise_low, noise_high, Y.shape)

    coef_lo  = -2
    coef_hi  =  2
    coef_mat = np.random.uniform(coef_lo, coef_hi, (batch_size, len(functions))) # creating a matrix of coefficients
    coef_mat = np.where(np.abs(coef_mat) < 10**-1, 0, coef_mat)

    for i in range(batch_size):

        curr_res = np.zeros((channels, sample_length * intervals))
        for func_id, function in enumerate(functions):
            curr_func = functions[func_id]
            curr_coef = coef_mat[i][func_id]
            curr_res += curr_coef * curr_func(Y[i, :, :])

        Y[i, :, :] = curr_res

    clean = Y
    noisy = clean + noise

    # Normalizing
    clean -= clean.min(axis=2, keepdims=2)
    clean /= clean.max(axis=2, keepdims=2) + 1e-5 #avoiding zero division

    noisy -= noisy.min(axis=2, keepdims=2)
    noisy /= noisy.max(axis=2, keepdims=2) + 1e-5 #avoiding zero division

    clean = torch.from_numpy(clean)
    noisy = torch.from_numpy(noisy)

    return x_axis, clean, noisy

Training loop:
functions = [lambda x: np.cos(0.1*x),
             lambda x: np.cos(x),
             lambda x: np.cos(3*x)]

num_epochs = 200

lin_loss_list = []

criterion = torch.nn.MSELoss()

lin_optimizer = torch.optim.SGD(lin_model.parameters(), lr=0.01, momentum=0.9)

_, val_clean, val_noisy = util.lincomb_generate_data(batch_size, B, K, functions, noise_type="gaussian")

print("STARTED TRAINING")

for epoch in range(num_epochs):
    # generate data returns the x-axis used for plotting as well as the clean and noisy data
    _, t_clean, t_noisy = util.lincomb_generate_data(batch_size, B, K, functions, noise_type="gaussian")

    # ===================forward=====================
    lin_output = lin_model(t_noisy.float())
    lin_loss = criterion(lin_output.float(), t_clean.float())
    lin_loss_list.append(lin_loss.data)

    # ===================backward====================
    lin_optimizer.zero_grad()
    lin_loss.backward()
    lin_optimizer.step()

    val_lin_loss = F.mse_loss(lin_model(val_noisy.float()), val_clean.float())

print("DONE TRAINING")

edit: shared the parameters requested
L = 1
K = 512
B = 2

batch_size = 64
z_dim = 64

noise_mean = 0
noise_std  = 0.4

Output: The problem was I didnt use nn.BatchNorm1d in my model so i guess something wrong happened during training (probably vanishing gradients).


Instruction: Shouldn't `randperm` in the PyTorch C++ API return a tensor with default type int?

Input:
When I try to generate a list of permuted integer indices with randperm using the C++ PyTorch API, the resulting tensor has the element type of CPUFloatType{10} instead of an integer type:
int N_SAMPLES = 10;
torch::Tensor shuffled_indices = torch::randperm(N_SAMPLES);
cout << shuffled_indices << endl;

returns
 9
 3
 8
 6
 2
 5
 4
 7
 1
 0
[ CPUFloatType{10} ]

Which cannot be used used for indexing of tensors because the element type is float and not an integer type. When tryig to use my_tensor.index(shuffled_indices) I get
terminate called after throwing an instance of 'c10::IndexError'
  what():  tensors used as indices must be long, byte or bool tensors

Environment:

python-pytorch, version 1.6.0-2 on Arch Linux
g++ (GCC) 10.1.0

Why does this happen?

Output:
That's because the default type of any tensor that you create with torch is always float. If you want otherwise, you have to specify it with the TensorOptions parameter struct :
int N_SAMPLES = 10;
torch::Tensor shuffled_indices = torch::randperm(N_SAMPLES, torch::TensorOptions().dtype(at::kLong));
cout << shuffled_indices.dtype() << endl;
>>> long


Instruction: Why does regularization in pytorch and scratch code does not match and what is the formula used for regularization in pytorch?

Input:
I have been trying to do L2 regularization on a binary classification model in PyTorch but when I match the results of PyTorch and scratch code it doesn't match,
Pytorch code:
class LogisticRegression(nn.Module):
  def __init__(self,n_input_features):
    super(LogisticRegression,self).__init__()
    self.linear=nn.Linear(4,1)
    self.linear.weight.data.fill_(0.0)
    self.linear.bias.data.fill_(0.0)

  def forward(self,x):
    y_predicted=torch.sigmoid(self.linear(x))
    return y_predicted

model=LogisticRegression(4)

criterion=nn.BCELoss()
optimizer=torch.optim.SGD(model.parameters(),lr=0.05,weight_decay=0.1)
dataset=Data()
train_data=DataLoader(dataset=dataset,batch_size=1096,shuffle=False)

num_epochs=1000
for epoch in range(num_epochs):
  for x,y in train_data:
    y_pred=model(x)
    loss=criterion(y_pred,y)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

Scratch Code:
def sigmoid(z):
    s = 1/(1+ np.exp(-z))
    return s

def yinfer(X, beta):
  return sigmoid(beta[0] + np.dot(X,beta[1:]))

def cost(X, Y, beta, lam):
    sum = 0
    sum1 = 0
    n = len(beta)
    m = len(Y)
    for i in range(m):
        sum = sum + Y[i]*(np.log( yinfer(X[i],beta)))+ (1 -Y[i])*np.log(1-yinfer(X[i],beta))
    for i in range(0, n):
        sum1 = sum1 + beta[i]**2

    return  (-sum + (lam/2) * sum1)/(1.0*m)

def pred(X,beta):
  if ( yinfer(X, beta) > 0.5):
    ypred = 1
  else :
    ypred = 0
  return ypred

beta = np.zeros(5)
iterations = 1000
arr_cost = np.zeros((iterations,4))
print(beta)
n = len(Y_train)
for i in range(iterations):
    Y_prediction_train=np.zeros(len(Y_train))
    Y_prediction_test=np.zeros(len(Y_test))

    for l in range(len(Y_train)):
        Y_prediction_train[l]=pred(X[l,:],beta)

    for l in range(len(Y_test)):
        Y_prediction_test[l]=pred(X_test[l,:],beta)

    train_acc = format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100)
    test_acc = 100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100
    arr_cost[i,:] = [i,cost(X,Y_train,beta,lam),train_acc,test_acc]
    temp_beta = np.zeros(len(beta))

    ''' main code from below '''

    for j in range(n):
        temp_beta[0] = temp_beta[0] + yinfer(X[j,:], beta) - Y_train[j]
        temp_beta[1:] = temp_beta[1:] + (yinfer(X[j,:], beta) - Y_train[j])*X[j,:]

    for k in range(0, len(beta)):
        temp_beta[k] = temp_beta[k] +  lam * beta[k]  #regularization here

    temp_beta= temp_beta / (1.0*n)

    beta = beta - alpha*temp_beta

graph of the losses
graph of training accuracy
graph of testing accuracy
Can someone please tell me why this is happening?
L2 value=0.1

Output:
Great question. I dug a lot through PyTorch documentation and found the answer. The answer is very tricky. Basically there are two ways to calculate regulalarization. (For summery jump to the last section).

The PyTorch uses the first type (in which regularization factor is not divided by batch size).
Here's a sample code which demonstrates that:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import torch.optim as optim

class model(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(1, 1)
        self.linear.weight.data.fill_(1.0)
        self.linear.bias.data.fill_(1.0)

    def forward(self, x):
        return self.linear(x)

model     = model()
optimizer = optim.SGD(model.parameters(), lr=0.1, weight_decay=1.0)

input     = torch.tensor([[2], [4]], dtype=torch.float32)
target    = torch.tensor([[7], [11]], dtype=torch.float32)

optimizer.zero_grad()
pred      = model(input)
loss      = F.mse_loss(pred, target)

print(f'input: {input[0].data, input[1].data}')
print(f'prediction: {pred[0].data, pred[1].data}')
print(f'target: {target[0].data, target[1].data}')

print(f'\nMSEloss: {loss.item()}\n')

loss.backward()

print('Before updation:')
print('--------------------------------------------------------------------------')
print(f'weight [data, gradient]: {model.linear.weight.data, model.linear.weight.grad}')
print(f'bias [data, gradient]: {model.linear.bias.data, model.linear.bias.grad}')
print('--------------------------------------------------------------------------')

optimizer.step()

print('After updation:')
print('--------------------------------------------------------------------------')
print(f'weight [data]: {model.linear.weight.data}')
print(f'bias [data]: {model.linear.bias.data}')
print('--------------------------------------------------------------------------')

which outputs:
input: (tensor([2.]), tensor([4.]))
prediction: (tensor([3.]), tensor([5.]))
target: (tensor([7.]), tensor([11.]))

MSEloss: 26.0

Before updation:
--------------------------------------------------------------------------
weight [data, gradient]: (tensor([[1.]]), tensor([[-32.]]))
bias [data, gradient]: (tensor([1.]), tensor([-10.]))
--------------------------------------------------------------------------
After updation:
--------------------------------------------------------------------------
weight [data]: tensor([[4.1000]])
bias [data]: tensor([1.9000])
--------------------------------------------------------------------------

Here m = batch size = 2, lr = alpha = 0.1, lambda = weight_decay = 1.
Now consider tensor weight which has value = 1 and grad = -32
case1(type1 regularization):
 weight = weight - lr(grad + weight_decay.weight)
 weight = 1 - 0.1(-32 + 1(1))
 weight = 4.1

case2(type2 regularization):
 weight = weight - lr(grad + (weight_decay/batch size).weight)
 weight = 1 - 0.1(-32 + (1/2)(1))
 weight = 4.15

From the output we can see that updated weight = 4.1000. That concludes PyTorch uses type1 regularization.
So finally In your code you are following type2 regularization. So just change some last lines to this:
# for k in range(0, len(beta)):
#    temp_beta[k] = temp_beta[k] +  lam * beta[k]  #regularization here

temp_beta= temp_beta / (1.0*n)

beta = beta - alpha*(temp_beta + lam * beta)

And also PyTorch loss functions doesn't include regularization term(implemented inside optimizers) so also remove regularization terms inside your custom cost function.
In summary:

Pytorch use this Regularization function:

Regularization is implemented inside Optimizers (weight_decay parameter).

PyTorch Loss functions doesn't include Regularization term.

Bias is also regularized if Regularization is used.

To use Regularization try:
torch.nn.optim.optimiser_name(model.parameters(), lr, weight_decay=lambda).