Instruction: PyTorch MNIST Model RuntimeError: mat1 and mat2 shapes cannot be multiplied (3584x28 and 784x10)

Input:
I'm trying to build a simple MNIST Model and this is what I've built -
training_loader = DataLoader(training_dataset, 128, shuffle = True)
validation_loader = DataLoader(validation_dataset, 128)

class mnistmodel(nn.Module):
  def __init__(self):
    super().__init__()
    self.linear1 = nn.Linear(784, 10)
    self.linear2 = nn.Linear(10, 5)
    self.linear3 = nn.Linear(5, 10)

  def forward(self, xb):
    xb.reshape(-1, 784)
    predicted = F.relu(self.linear1(xb))
    predicted.reshape(-1, 10)
    predicted = F.relu(self.linear2(predicted))
    predicted.reshape(-1, 5)
    predicted = self.linear3(predicted)
    return predicted

  def training_step(self, batch):
    images, labels = batch
    predicted = self(images)
    loss = F.cross_entropy(predicted, labels)
    return loss

  def validation_step(self, batch):
    images, labels = batch
    predicted = self(images)
    loss = F.cross_entropy(predicted, labels)
    _, preds = torch.max(predicted, dim=1)
    accuracy = torch.tensor(torch.sum(preds == labels).item() / len(preds))
    return {'validation_loss': loss, 'validation_accuracy': accuracy}

  def validation_epoch_end(self, outputs):
    batch_losses = [x['validation_loss'] for x in outputs]
    epoch_loss = torch.stack(batch_losses).mean()
    batch_accs = [x['validation_acc'] for x in outputs]
    epoch_acc = torch.stack(batch_accs).mean()
    return {'validation_loss': epoch_loss.item(), 'validation_accuracy': epoch_acc.item()}

def epoch_end(self, epoch, result):
    print(f"Epoch [{epoch}], val_loss: {result['validation_loss']}, val_acc: {result['validation_acc']}")

model = mnistmodel()

def fit_mnist(epochs, lr, model, training_loader, validation_loader, optimizer_function=torch.optim.SGD):
    optimizer = optimizer_function(model.parameters(), lr)
    history = []

    for epoch in range(epochs):

        for batch in training_loader:
            loss = model.training_step(batch)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()

        result = evaluate(model, validation_loader)
        model.epoch_end(epoch, result)
        history.append(result)

    return history

history1 = fit_mnist(5, 0.001, model, training_loader, validation_loader)

I get the following error -
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-162-48e4fe0cc2d9> in <module>()
----> 1 history1 = fit_mnist(5, 0.001, model, training_loader, validation_loader)

6 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
   1751     if has_torch_function_variadic(input, weight):
   1752         return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753     return torch._C._nn.linear(input, weight, bias)
   1754
   1755

RuntimeError: mat1 and mat2 shapes cannot be multiplied (3584x28 and 784x10)

I'm new to pytorch but as far as I understand the shapes seem to be fine, what is going wrong here?
I've checked manually and the dimensions seem to be fine. I even tried multiple things removing adding reshape commands but this error persists

Output: I've found the mistake I need to reassign the reshape which I missed and spent the rest of the time debugging.


Instruction: ValueError: Expected more than 1 value per channel when training in pytorch 1.8

Input:
The input tensor size is [1, 512] and I feed it in a GroupNorm layer.
The layer is self.fc_bn1 = nn.GroupNorm(512 // reduction, 512 // reduction) and the input is x = self.fc_bn1(x)
The error is visible in this image:

That will be worked when I run it in pytorch 1.1, but now I got a machine with pytorch 1.8 and it doesn't work. Since GPUs in this machine cannot use pytorch 1.1, I want to know how to solve this.
I already set model.eval(), so I guess it's not this reason.

Output: This code does work in pytorch1.1. But I think this is a bug, because the output after GroupNorm is all 0, which is consistent with the mathematical formula. In subsequent versions, this bug has been fixed, so an error is reported.


Instruction: pytorch cyclegann gives a Missing key error when testing

Input:
I have trained a model using the pix2pix pytorch implementation and would like to test it.
However when I test it I get the error
model [CycleGANModel] was created
loading the model from ./checkpoints/cycbw50/latest_net_G_A.pth
Traceback (most recent call last):
  File "test.py", line 47, in <module>
    model.setup(opt)               # regular setup: load and print networks; create schedulers
  File "/media/bitlockermount/SmartImageToDigitalTwin/SmartImageToDigitalTwin/bin/python/cyclegann/pytorch-CycleGAN-and-pix2pix/models/base_model.py", line 88, in setup
    self.load_networks(load_suffix)
  File "/media/bitlockermount/SmartImageToDigitalTwin/SmartImageToDigitalTwin/bin/python/cyclegann/pytorch-CycleGAN-and-pix2pix/models/base_model.py", line 199, in load_networks
    net.load_state_dict(state_dict)
  File "/home/bst/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 846, in load_state_dict
    raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
RuntimeError: Error(s) in loading state_dict for ResnetGenerator:
    Missing key(s) in state_dict: "model.1.bias", "model.4.bias", "model.7.bias", "model.10.conv_block.1.bias", "model.10.conv_block.5.bias", "model.11.conv_block.1.bias", "model.11.conv_block.5.bias", "model.12.conv_block.1.bias", "model.12.conv_block.5.bias", "model.13.conv_block.1.bias", "model.13.conv_block.5.bias", "model.14.conv_block.1.bias", "model.14.conv_block.5.bias", "model.15.conv_block.1.bias", "model.15.conv_block.5.bias", "model.16.conv_block.1.bias", "model.16.conv_block.5.bias", "model.17.conv_block.1.bias", "model.17.conv_block.5.bias", "model.18.conv_block.1.bias", "model.18.conv_block.5.bias", "model.19.bias", "model.22.bias".
    Unexpected key(s) in state_dict: "model.2.weight", "model.2.bias", "model.5.weight", "model.5.bias", "model.8.weight", "model.8.bias", "model.10.conv_block.2.weight", "model.10.conv_block.2.bias", "model.10.conv_block.6.weight", "model.10.conv_block.6.bias", "model.11.conv_block.2.weight", "model.11.conv_block.2.bias", "model.11.conv_block.6.weight", "model.11.conv_block.6.bias", "model.12.conv_block.2.weight", "model.12.conv_block.2.bias", "model.12.conv_block.6.weight", "model.12.conv_block.6.bias", "model.13.conv_block.2.weight", "model.13.conv_block.2.bias", "model.13.conv_block.6.weight", "model.13.conv_block.6.bias", "model.14.conv_block.2.weight", "model.14.conv_block.2.bias", "model.14.conv_block.6.weight", "model.14.conv_block.6.bias", "model.15.conv_block.2.weight", "model.15.conv_block.2.bias", "model.15.conv_block.6.weight", "model.15.conv_block.6.bias", "model.16.conv_block.2.weight", "model.16.conv_block.2.bias", "model.16.conv_block.6.weight", "model.16.conv_block.6.bias", "model.17.conv_block.2.weight", "model.17.conv_block.2.bias", "model.17.conv_block.6.weight", "model.17.conv_block.6.bias", "model.18.conv_block.2.weight", "model.18.conv_block.2.bias", "model.18.conv_block.6.weight", "model.18.conv_block.6.bias", "model.20.weight", "model.20.bias", "model.23.weight", "model.23.bias".

The opt file for the training:
----------------- Options ---------------
               batch_size: 1
                    beta1: 0.5
          checkpoints_dir: ./checkpoints
           continue_train: False
                crop_size: 256
                 dataroot: ./datasets/datasets/boundedwalls_50_0.1/ [default: None]
             dataset_mode: aligned                          [default: unaligned]
                direction: AtoB
              display_env: main
             display_freq: 400
               display_id: 1
            display_ncols: 4
             display_port: 8097
           display_server: http://localhost
          display_winsize: 256
                    epoch: latest
              epoch_count: 1
                 gan_mode: lsgan
                  gpu_ids: 0
                init_gain: 0.02
                init_type: normal
                 input_nc: 3
                  isTrain: True                             [default: None]
                 lambda_A: 10.0
                 lambda_B: 10.0
          lambda_identity: 0.5
                load_iter: 0                                [default: 0]
                load_size: 286
                       lr: 0.0002
           lr_decay_iters: 50
                lr_policy: linear
         max_dataset_size: inf
                    model: cycle_gan
                 n_epochs: 100
           n_epochs_decay: 100
               n_layers_D: 3
                     name: cycbw50                          [default: experiment_name]
                      ndf: 64
                     netD: basic
                     netG: resnet_9blocks
                      ngf: 64
               no_dropout: True
                  no_flip: False
                  no_html: False
                     norm: batch                            [default: instance]
              num_threads: 4
                output_nc: 3
                    phase: train
                pool_size: 50
               preprocess: resize_and_crop
               print_freq: 100
             save_by_iter: False
          save_epoch_freq: 5
         save_latest_freq: 5000
           serial_batches: False
                   suffix:
         update_html_freq: 1000
                  verbose: False
----------------- End -------------------

And when testing it I used the following testing settings
python3 test.py --dataroot ./datasets/datasets/boundedwalls_50_0.1 --name cycbw50 --model pix2pix --netG resnet_9blocks --direction BtoA --dataset_mode aligned --norm batch --load_size 286

----------------- Options ---------------
             aspect_ratio: 1.0
               batch_size: 1
          checkpoints_dir: ./checkpoints
                crop_size: 256
                 dataroot: ./datasets/datasets/boundedwalls_50_0.1/ [default: None]
             dataset_mode: unaligned
                direction: AtoB
          display_winsize: 256
                    epoch: latest
                     eval: False
                  gpu_ids: 0
                init_gain: 0.02
                init_type: normal
                 input_nc: 3
                  isTrain: False                            [default: None]
                load_iter: 0                                [default: 0]
                load_size: 256
         max_dataset_size: inf
                    model: cycle_gan                        [default: test]
               n_layers_D: 3
                     name: cycbw50                          [default: experiment_name]
                      ndf: 64
                     netD: basic
                     netG: resnet_9blocks
                      ngf: 64
               no_dropout: True
                  no_flip: False
                     norm: instance
                 num_test: 50
              num_threads: 4
                output_nc: 3
                    phase: test
               preprocess: resize_and_crop
              results_dir: ./results/
           serial_batches: False
                   suffix:
                  verbose: False
----------------- End -------------------

Does anybody see what I'm doing wrong here? I would like to run this network such that I get results for individual images, so far the test function seems the most promising but it just crashes on this neural network.

Output:
I think the problem here is some layer the bias=None but in testing the model required this, you should check the code for details.
After I check your config in train and test, the norm is different. For the code in GitHub, the norm difference may set the bias term is True or False.
if type(norm_layer) == functools.partial:
   use_bias = norm_layer.func == nn.InstanceNorm2d
else:
   use_bias = norm_layer == nn.InstanceNorm2d

model = [nn.ReflectionPad2d(3),
         nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),
         norm_layer(ngf),
         nn.ReLU(True)]

You can check it here.


Instruction: XLM/BERT sequence outputs to pooled output with weighted average pooling

Input:
Let's say I have a tokenized sentence of length 10, and I pass it to a BERT model.
bert_out = bert(**bert_inp)
hidden_states = bert_out[0]
hidden_states.shape
>>>torch.Size([1, 10, 768])

This returns me a tensor of shape: [batch_size, seq_length, d_model] where each word in sequence is encoded as a 768-dimentional vector
In TensorFlow BERT also returns a so called pooled output which corresponds to a vector representation of a whole sentence.
I want to obtain it by taking a weighted average of sequence vectors and the way I do it is:
hidden_states.view(-1, 10).shape
>>> torch.Size([768, 10])

pooled = nn.Linear(10, 1)(hidden_states.view(-1, 10))
pooled.shape
>>> torch.Size([768, 1])

Is it the right way to proceed, or should I just flatten the whole thing and then apply linear?
Any other ways to obtain a good sentence representation?

Output:
There are two simple ways to get a sentence representation:

Get the vector for the CLS token.
Get the pooler_output

Assuming the input is [batch_size, seq_length, d_model], where batch_size is the number of sentences, then to get the CLS token for every sentence:
bert_out = bert(**bert_inp)
hidden_states = bert_out['last_hidden_state']
cls_tokens = hidden_states[:, 0, :]  # 0 for the CLS token for every sentence.

You will have a tensor with shape (batch_size, d_model).
To get the pooler_output:
bert_out = bert(**bert_inp)
pooler_output = bert_out['pooler_output']

Again you get a tensor with shape (batch_size, d_model).


Instruction: AttributeError: 'Model' object has no attribute 'parameters'

Input:
I am using a modified Resnet18, with my own pooling function at the end of the Resnet.
Here is my code:
resnet = resnet18().cuda() #a modified resnet

class Model():
    def __init__(self, model, pool):
        self.model = model
        self.pool= pool #my own pool class which has trainable layers

    def forward(self, sample):
        output = self.model(sample)
        output = self.pool(output)
        output = F.normalize(output, p=2, dim=1)
        return output

Now, obviously I need to train not only the resnet part, but also the pool part.
But, when I check:
model = Model(model=resnet, pool= pool)
print(list(model.parameters()))

It gives:
AttributeError: 'Model' object has no attribute 'parameters'

Can anyone help?

Output:
You need you Model to inherit torch.nn.Module:
class Model(torch.nn.Module):
    def __init__(self, model, pool):
        super(Model, self).__init__()
        ...


Instruction: In Pytorch how to slice tensor across multiple dims with BoolTensor masks?

Input:
I want to use BoolTensor indices to slice a multidimensional tensor in Pytorch. I expect for the indexed tensor, the parts where the indices are true are kept, while the parts where the indices are false are sliced out.
My code is like
import torch
a = torch.zeros((5, 50, 5, 50))

tr_indices = torch.zeros((50), dtype=torch.bool)
tr_indices[1:50:2] = 1
val_indices = ~tr_indices

print(a[:, tr_indices].shape)
print(a[:, tr_indices, :, val_indices].shape)

I expect a[:, tr_indices, :, val_indices] to be of shape [5, 25, 5, 25], however it returns [25, 5, 5]. The result is
torch.Size([5, 25, 5, 50])
torch.Size([25, 5, 5])

I'm very confused. Can anyone explain why?

Output:
PyTorch inherits its advanced indexing behaviour from Numpy. Slicing twice like so should achieve your desired output:
a[:, tr_indices][..., val_indices]


Instruction: Vectorize for-loop - need to average slices of varying size

Input:
I am trying to average subword embeddings to form a word-level representation. Each word has a corresponding start and end index, indicating which subwords make up that word.
sequence_output is a tensor of B * 3 * 2, where 3 is the max sequence length, and 2 is the number of features.
all_token_mapping is a tensor of B * 3 * 2, which contains a start and end index.
initial_reps is a tensor of num_nodes * 2, num_nodes is the sum of all the number of words (not subwords) in the different samples.
sequence_output = torch.arange(2*3*2).float().reshape(2, 3, 2)
tensor([[[ 0.,  1.],
         [ 2.,  3.],
         [ 4.,  5.]],

        [[ 6.,  7.],
         [ 8.,  9.],
         [10., 11.]]])

all_token_mapping = torch.tensor([[[0,0],[1,2],[-1,-1]], [[0,2],[-1,-1],[-1,-1]]])
tensor([[[ 0,  0],
         [ 1,  2],
         [-1, -1]],

        [[ 0,  2],
         [-1, -1],
         [-1, -1]]])

num_nodes = 0
for sample in all_token_mapping:
  for mapping in sample:
    if mapping[0] != -1:
      num_nodes += 1
3

initial_reps = torch.empty((num_nodes, 2), dtype=torch.float32)
current_idx = 0
for i, feature_tokens_mapping in enumerate(all_token_mapping):
    for j, token_mapping in enumerate(feature_tokens_mapping):
        if token_mapping[0] == -1: # reached the end for this particular sequence
            break
        initial_reps[current_idx] = torch.mean(sequence_output[i][token_mapping[0]:token_mapping[-1] + 1], 0, keepdim=True)
        current_idx += 1
initial_reps
tensor([[0., 1.],
        [3., 4.],
        [8., 9.]])

In the example above, initial_reps[0] will be the mean of sequence_output[0][0:1], initial_reps[1] will be the mean of sequence_output[0][1:3], and initial_reps[2] will be the mean of sequence_output[1][0:3].
My current code will create an empty tensor of length num_nodes, and a for loop will calculate the values at each index, by checking token_mapping[0] and token_mapping[1] for the correct slice of sequence_output to average.
Is there a way to vectorize this code?
In addition, I have a list that holds the number of words for each sample. i.e. the sum of all the elements in the list == num_nodes

Output:
I figured out a way with the help of someone over at https://discuss.pytorch.org/t/vectorize-for-loop-need-to-average-slices-of-varying-size/122618/2
initial_reps_list = []
for i, sample_output in enumerate(sequence_output):
    token_mapping = all_token_mapping[i]
    token_mapping = token_mapping[token_mapping != -1]
    non_padded_outputs = sample_output[:num_bert_tokens[i]]
    initial_reps_list.append(torch_scatter.segment_coo(non_padded_outputs, token_mapping, reduce="mean"))

initial_reps = torch.cat(initial_reps_list)

token_mapping is a list of indices in ascending order up to the max sequence length, padded with -1. I loop through the batch, for each sample, I get the token mapping, and only keep the non-negative indices.
num_bert_tokens is a list that holds, for each sample, the number of tokens (no padding). I get the non-padded outputs, use segment_coo to reduce them according to the token_mapping, and append them all to a list.
After the loop, I concatenate all the tensors in the list together.
The method segment_coo reduces all values from the src tensor into out at the indices specified in the index tensor along the last dimension of index. More details can be found at: https://pytorch-scatter.readthedocs.io/en/latest/functions/segment_coo.html
Runs much faster now!


Instruction: Rearranging PyTorch tensor in a windowed manner

Input:
So I have a tensor
A = torch.tensor([[1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16]])

and I was hoping to rearrange it in sliding windows, that is,
f(A) = torch.tensor([[1,2,5,6],[3,4,7,8],[9,10,13,14],[11,12,15,16]])

by viewing every tensor in A as a 2-by-2 small window, and then arranging the "window"s in a 2-by-2 matrix. This operation is similar to ArrayFlatten of Mathematica, but I couldn' find a way to do it in PyTorch. Any help is welcome.

Output:
I can't think of a neat way to do this off the top of my head, but you can achieve it with a few choice slices and concatenations:
A_ = A.reshape(4,2,2)
torch.cat([torch.cat([*A_[:2]],1), torch.cat([*A_[2:]],1)],0)

Alternative approaches:
A.unfold(1,2,2).unfold(0,2,2).reshape(*A.shape).index_select(1, torch.LongTensor([0,2,1,3]))

torch.cat([*A_],1).reshape(*A.shape).index_select(0, torch.LongTensor([0,2,1,3]))


Instruction: How to write a fast PyTorch training loop?

Input:
I've encountered a bad regression in performance after porting a machine learning algorithm from Keras to PyTorch.
The following construction in Keras:
model.compile(loss="binary_crossentropy", optimizer=SGD(learning_rate=0.01))
.
.
.
model.fit(states, actions)

is ~15x faster than the following verbose version in Pytorch:
def train_network(model, optimizer, train_loader,
                  num_epochs=16, pbar_update_interval=200, print_logs=True):

    criterion = nn.BCELoss()
    model.train()

    for i in range(num_epochs):
        for k, batch_data in enumerate(train_loader):
            optimizer.zero_grad()

            batch_x = batch_data[:, :-1]
            batch_y = batch_data[:, -1]

            y_pred = model(batch_x)
            loss = criterion(y_pred, batch_y.unsqueeze(1))
            loss.backward()
            optimizer.step()

optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)
train_data = torch.from_numpy(np.column_stack((states, actions)))
train_data = train_data.to(torch.float)
train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=32)
train_network(model, optimizer, train_loader)

Am I writing the PyTorch code wrong?
The variable model in the PyTorch code is an nn.Module, and states and actions are NumPy arrays. The code is running on 4 CPU cores.

Output: It seems that keras.model.fit has default parameter value epochs=1. So this exactly explains the performance factor.


Instruction: How to extract tensors to numpy arrays or lists from a larger pytorch tensor

Input:
I have a list of pytorch tensors as shown below:
data = [[tensor([0, 0, 0]), tensor([1, 2, 3])],
        [tensor([0, 0, 0]), tensor([4, 5, 6])]]

Now this is just a sample data, the actual one is quite large but the structure is similar.
Question: I want to extract the tensor([1, 2, 3]), tensor([4, 5, 6]) i.e., the index 1 tensors from data to either a numpy array or a list in flattened form.
Expected Output:
out = array([1, 2, 3, 4, 5, 6])

OR
out = [1, 2, 3, 4, 5, 6]

I have tried several ways one including map function like:
map(lambda x: x[1].numpy(), data)

This gives:
[array([1, 2, 3]),
 array([4, 5, 6])]

And I'm unable to get the desired result with any other method I'm using.

Output:
OK, you can just do this.
out = np.concatenate(list(map(lambda x: x[1].numpy(), data)))


Instruction: Creating a custom BCE with logit loss function

Input:
I want to create a custom loss function for multi-label classification. The idea is to weigh the positive and negative labels differently. For this, I am making use of this custom code implementation.
class WeightedBCEWithLogitLoss(nn.Module):
    def __init__(self, pos_weight, neg_weight):
        super(WeightedBCEWithLogitLoss, self).__init__()
        self.register_buffer('neg_weight', neg_weight)
        self.register_buffer('pos_weight', pos_weight)

    def forward(self, input, target):
        assert input.shape == target.shape, "The loss function received invalid input shapes"
        y_hat = torch.sigmoid(input + 1e-8)
        loss = -1.0 * (self.pos_weight * target * torch.log(y_hat + 1e-6) + self.neg_weight * (1 - target) * torch.log(1 - y_hat + 1e-6))
        # Account for 0 times inf which leads to nan
        loss[torch.isnan(loss)] = 0
        # We average across each of the extra attribute dimensions to generalize it
        loss = loss.mean(dim=1)
        # We use mean reduction for our task
        return loss.mean()

I started getting nan values which I realized happened because of 0 times inf multiplication. I handled it as shown in the figure. Next, I again saw getting inf as the error value and corrected it by adding 1e-6 to the log (I tried with 1e-8 but that still gave me inf error value).
It would be great if someone can take a look and suggest further improvements and rectify any more bugs visible here.

Output:
You don't have to implement it. It is already done.
The BCEWithLogits accepts parameter pos_weight which, according to documentation, is used as
pos_weight ( Tensor , optional ) – a weight of positive examples.
Must be a vector with length equal to the number of classes


Instruction: Execution block when calling DataLoader with self-defined functions on the dataset class

Input:
I am running my code in Colab. I use Dataloader to load my dataset as batches for training and validation, and tqdm to visualize the training process. But every time I execute my code, I get stuck at a random point (in epoch 0, meaning the dataset has never been traversed), saying that the program is still running but the progress bar is frozen. An interesting phenomenon lies in the self-defined transformation functions of the inputs since once I drop these transformations, everything went smoothly. The functions are:
class Sharpen(object):
    def __init__(self, p=0.5):
        self.p = p

    def __call__(self, sample):
        if random.uniform(0.0, 1.0) < self.p:
            return sample
        kernel = np.array([[0, -1, 0], [-1, 5, -1], [0, -1, 0]], np.float32)
        for i in range(len(sample['img_list'])):
            sample['img_list'][i] = cv2.filter2D(sample['img_list'][i], -1, kernel=kernel)

        return sample

class Rotation(object):
    def __init__(self, angle=5, fill_value=0, p=0.5):
        self.angle = angle
        self.fill_value = fill_value
        self.p = p

    def __call__(self, sample):
        if random.uniform(0.0, 1.0) < self.p:
            return sample
        for i in range(len(sample['img_list'])):
            ang_rot = np.random.uniform(self.angle) - self.angle / 2
            h, w, _ = sample["img_list"][i].shape
            if h*w == 0:
                continue
            transform = cv2.getRotationMatrix2D((w / 2, h / 2), ang_rot, 1)
            sample["img_list"][i] = cv2.warpAffine(sample["img_list"][i], transform, (w, h),
                              borderValue=self.fill_value)

        return sample

class Translation(object):
    def __init__(self, fill_value=0, p=0.5):
        self.fill_value = fill_value
        self.p = p
        self.SIGMA = 1e-3

    def __call__(self, sample):
        rand_num = random.uniform(0.0, 1.0)
        if rand_num <= self.p:
            return sample

        for i in range(len(sample['img_list'])):
            h, w, _ = sample["img_list"][i].shape
            if h*w == 0:
                continue
            trans_range = (w / 10, h / 10)
            tr_x = trans_range[0] * rand_num - trans_range[0] / 2 + self.SIGMA
            tr_y = trans_range[1] * rand_num - trans_range[1] / 2 + self.SIGMA
            transform = np.float32([[1, 0, tr_x], [0, 1, tr_y]])
            sample["img_list"][i] = cv2.warpAffine(sample["img_list"][i], transform, (w, h),
                                borderValue=self.fill_value)

        return sample

class Normalization(object):
    def __init__(self, mean=(0,0,0), std=(255,255,255)):
        self.mean = mean
        self.std = std

    def __call__(self, sample):
        # norm_func = transforms.Normalize(self.mean, self.std)
        for i in range(len(sample['img_list'])):
            for j in range(3):  # for colored image
                sample['img_list'][i][:,:,j] = np.array(list(map(lambda x: (x-self.mean[j])/self.std[j], sample['img_list'][i][:,:,j])),
                                     dtype=np.float32)
            # sample['img_list'][i] = norm_func(torch.Tensor(sample['img_list'][i]))
        return sample

The parameter sample is a dictionary, and the key: img_list represents a list of images (numpy.ndarray) processed by cv2.imread. The shape of each image is (h, w, c) and the size is not equal across images.
I created a Dataset Class and proceeded input images with Compose([Sharpen(), Rotation(), Translation(), Normalization()]). Now I don't know what's wrong with the transformation code. Any kind of help is appreciated!

Output: Problem solved. It is because of the data. If the input image is an empty one, the loading procedure will get stuck.


Instruction: Downloading Big Packages using conda in batches

Input:
I am curious whether I can download big packages in batches using conda (like in web browsers where we can resume interrupted downloads).
I wanted to update my pytorch package but the size is too big for my limited mobile data.
The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    libuv-1.40.0               |       he774522_0         255 KB
    pytorch-1.8.1              |py3.7_cuda10.1_cudnn7_0 836.4 MB  pytorch
    torchvision-0.9.1          |       py37_cu101         7.3 MB  pytorch
    ------------------------------------------------------------
                                           Total:       843.9 MB

The following NEW packages will be INSTALLED:

  libuv              pkgs/main/win-64::libuv-1.40.0-he774522_0

The following packages will be UPDATED:

  pytorch                      1.4.0-py3.7_cuda101_cudnn7_0 --> 1.8.1-py3.7_cuda10.1_cudnn7_0
  torchvision                              0.5.0-py37_cu101 --> 0.9.1-py37_cu101

Output:
No this is not possible in conda. Completely downloaded files will not be redownloaded, but files which are only partially downloaded before interruption will need to be redownloaded:
https://github.com/conda/conda/issues/800


Instruction: How to check the output gradient by each layer in pytorch in my code?

Input:
I am working on the pytorch to learn.
And  There is a question how to check the output gradient by each layer in my code.
My code is below
#import the nescessary libs
import numpy as np
import torch
import time

# Loading the Fashion-MNIST dataset
from torchvision import datasets, transforms

# Get GPU Device

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.5,), (0.5,))
                                                                   ])
# Download and load the training data
trainset = datasets.FashionMNIST('MNIST_data/', download = True, train = True, transform = transform)
testset = datasets.FashionMNIST('MNIST_data/', download = True, train = False, transform = transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size = 32, shuffle = True, num_workers=4)
testloader = torch.utils.data.DataLoader(testset, batch_size = 32, shuffle = True, num_workers=4)

# Examine a sample
dataiter = iter(trainloader)
images, labels = dataiter.next()

# Define the network architecture
from torch import nn, optim
import torch.nn.functional as F

model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 10),
                      nn.LogSoftmax(dim = 1)
                     )
model.to(device)

# Define the loss
criterion = nn.CrossEntropyLoss()

# Define the optimizer
optimizer = optim.Adam(model.parameters(), lr = 0.001)

# Define the epochs
epochs = 5

train_losses, test_losses = [], []

# start = time.time()
for e in range(epochs):
    running_loss = 0

    for images, labels in trainloader:
    # Flatten Fashion-MNIST images into a 784 long vector
        images = images.to(device)
        labels = labels.to(device)
        images = images.view(images.shape[0], -1)

    # Training pass
        optimizer.zero_grad()

        output = model.forward(images)
        loss = criterion(output, labels)

        loss.backward()

#         print(loss.grad)

        optimizer.step()

        running_loss += loss.item()

    else:
        print(model[0].grad)

If I print model[0].grad after back-propagation, Is it going to be the output gradient by each layer for every epoches?
Or, If I want to know the output gradient by each layer, where and what am I should print?
Thank you!!
Thank you for reading

Output:
Well, this is a good question if you need to know the inner computation within your model. Let me explain to you!
So firstly when you print the model variable you'll get this output:
Sequential(
  (0): Linear(in_features=784, out_features=128, bias=True)
  (1): ReLU()
  (2): Linear(in_features=128, out_features=10, bias=True)
  (3): LogSoftmax(dim=1)
)

And if you choose model[0], that means you have selected the first layer of the model. that is Linear(in_features=784, out_features=128, bias=True). If you will look at the documentation of torch.nn.Linear here, you will find that there are two variables to this class that you can access. One is Linear.weight and the other is Linear.bias which will give you the weights and biases of that corresponding layer respectively.
Remember you cannot use model.weight to look at the weights of the model as your linear layers are kept inside a container called nn.Sequential which doesn't has a weight attribute.
So coming back to looking at weights and biases, you can access them per layer. So model[0].weight and model[0].bias are the weights and biases of the first layer. And similarly to access the gradients of the first layer model[0].weight.grad and model[0].bias.grad will be the gradients.


Instruction: tqdm notebook - 2 inner bars

Input:
I'm trying to print training progress bar using tqdm.
I'd like to track the progress of the epochs, and for each epoch i have 2 progress bars: the train_loader minibatches, and validation_loader minibatches.
The code is something like that:
for epoch in tqdm(range(epochs_num)):
    for inputs, labels in tqdm(train_loader, "Train progress", leave=False):
        # train...
    with torch.no_grad():
        for inputs, labels in tqdm(validation_loader, "Validation progress", leave=False):
            # calc validation loss

Using the leave argument, the progress bars deleted every epoch, but I'd like to delete them togather, right after the validation process ends.
There's any way to do it?
Thanks

Output:
You can reuse your progress bars and do the updates manually like this:
epochs = tqdm(range(epochs_num), desc="Epochs")
training_progress = tqdm(total=training_batch_size, desc="Training progress")
validation_progress = tqdm(total=validation_batch_size, desc="Validation progress")

for epoch in epochs:
    training_progress.reset()
    validation_progress.reset()

    for inputs, labels in train_loader:
        # train...
        training_progress.update()

    with torch.no_grad():
        for inputs, labels in validation_loader:
            # calc validation loss
            validation_progress.update()

If the batch sizes are not always the same, you can calculate them on the fly and call reset(total=new_size).


Instruction: How to access pytorch embeddings lookup table as a tensor

Input:
I want to show my embeddings with the tensorboard projector. I would like to access the embeddings matrix (lookup table) of one of my layers so I can write it to the logs.
I instantiate my layer as this:
self.embeddings_user = torch.nn.Embedding(30,300)
And I'm looking for the tensor with shape (30,300) of 30 users with embedding on 300 to dimensions to replace it with the vectors variable in this sample code:
import numpy as np
import tensorflow as tf
import tensorboard as tb
tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
from torch.utils.tensorboard import SummaryWriter

vectors = np.array([[0,0,1], [0,1,0], [1,0,0], [1,1,1]])
metadata = ['001', '010', '100', '111']  # labels
writer = SummaryWriter()
writer.add_embedding(vectors, metadata)
writer.close()

Output:
Embeddings layers have weight attributes corresponding to the lookup table. You can access it as follows.
vectors = self.embeddings_user.weight

So now you can visualize it with tensorboard.
import numpy as np
import tensorflow as tf
import tensorboard as tb
tf.io.gfile = tb.compat.tensorflow_stub.io.gfile
from torch.utils.tensorboard import SummaryWriter

vectors = self.embeddings_user.weight
metadata = ['001', '010', '100', '111', ...]  # labels
writer = SummaryWriter()
writer.add_embedding(vectors, metadata)
writer.close()


Instruction: Pytorch BCELoss function different outputs for same inputs

Input:
I am trying to calculate cross entropy loss using pytorch's BCELoss Function for a binary classification problem. While tinkering I found this weird behaviour.
from torch import nn
sigmoid = nn.Sigmoid()
loss = nn.BCELoss(reduction="sum")
target = torch.tensor([0., 1.])

input1 = torch.tensor([1., 1.], requires_grad=True)
input2 = sigmoid(torch.tensor([10., 10.], requires_grad=True))
print(input2) #tensor([1.0000, 1.0000], grad_fn=<SigmoidBackward>)

print(loss(input1, target)) #tensor(100., grad_fn=<BinaryCrossEntropyBackward>)
print(loss(input2, target)) #tensor(9.9996, grad_fn=<BinaryCrossEntropyBackward>)

Since both input1 and input2 have same value, shouldn't it return the same loss value instead of 100 and 9.9996. The correct loss value should be 100 since I am multiplying log(0) ~-infinity which is capped at -100 in pytorch. https://pytorch.org/docs/stable/generated/torch.nn.BCELoss.html
What is going on here and where am I going wrong?

Output:
sigmoid(10) is not exactly equal to 1:
>>> 1 / (1 + torch.exp(-torch.tensor(10.))).item()
0.9999545833234493

In your case:
>>> sigmoid(torch.tensor([10., 10.], requires_grad=True)).tolist()
[0.9999545812606812, 0.9999545812606812]

Thus input1 is not the same as input2: [1.0, 1.0] vs [0.9999545812606812, 0.9999545812606812],
Let's compute BCE manually:
def bce(x, y):
    return - (y * torch.log(x) + (1 - y) * torch.log(1 - x)).item()

# input1

x1 = torch.tensor(1.)
x2 = torch.tensor(1.)

y1 = torch.tensor(0.)
y2 = torch.tensor(1.)

print("input1:", sum([bce(x1, y1), bce(x2, y2)]))

# input2

x1 = torch.tensor(0.9999545812606812)
x2 = torch.tensor(0.9999545812606812)

y1 = torch.tensor(0.)
y2 = torch.tensor(1.)

print("input2:", sum([bce(x1, y1), bce(x2, y2)]))

input1: nan
input2: 9.999631525119185

For input1 we get nan, but according to docs:

Our solution is that BCELoss clamps its log function outputs to be greater than or equal to -100. This way, we can always have a finite loss value and a linear backward method.

That's why we have 100 in a final pytorch's BCE output.


Instruction: Pytorch data.random_split() doesn't split randomly

Input:
I am trying to split my custom dataset randomly into test and train. The code runs and outputs the test and train folders successfully, but I need the test and train sets to be different each time I ran the code. Isn't this what randomly splitting supposed to mean/do?
p.s. just for clarification, the data is images, so I expect to see different images chosen for test and train sets each time I execute the code.
#Set the random seeds for reproducibility
SEED = 1234

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True

TRAIN_RATIO = 0.9

data_dir = 'Data Set 1'
images_dir = os.path.join(data_dir, 'images')
train_dir = os.path.join(data_dir, 'train')
test_dir = os.path.join(data_dir, 'test')

if os.path.exists(train_dir):
    shutil.rmtree(train_dir)
if os.path.exists(test_dir):
    shutil.rmtree(test_dir)

os.makedirs(train_dir)
os.makedirs(test_dir)

classes = os.listdir(images_dir)

for c in classes:

    class_dir = os.path.join(images_dir, c)

    images = os.listdir(class_dir)

    n_train = int(len(images) * TRAIN_RATIO)
    n_test = len(images) - n_train
    train_images, test_images = data.random_split(images,
                                               [n_train, n_test])

    os.makedirs(os.path.join(train_dir, c), exist_ok=True)
    os.makedirs(os.path.join(test_dir, c), exist_ok=True)

    for image in train_images:
        image_src = os.path.join(class_dir, image)
        image_dst = os.path.join(train_dir, c, image)
        shutil.copyfile(image_src, image_dst)

    for image in test_images:
        image_src = os.path.join(class_dir, image)
        image_dst = os.path.join(test_dir, c, image)
        shutil.copyfile(image_src, image_dst)

Output:
It's not random because you set the random seed. Think of the seed as a random number - if you define the seed, the number is not random anymore. If you don't define it, you will get a random seed every time.
Just comment out these lines :)
SEED = 1234

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)

Alternatively, just do this: SEED = random.randint(1, 1000) to get a random number between 1 and 1000. This will let you print the value of SEED, if you need that for some reason.
You can see how the seed function works here:
https://www.w3schools.com/python/ref_random_seed.asp


Instruction: How to make prediction on pytorch emotion detection model

Input:
I made a CNN model for emotion recognition on 5 emotions. I wanted to test it on an single image to get the individual class predictions for each emotion.
Evaluating the model works, but I can't seem to find how to make a prediction with a single image. How can I do that?
The Model
def conv_block(in_channels, out_channels, pool=False):
layers = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
          nn.BatchNorm2d(out_channels),
          nn.ELU(inplace=True)]
if pool: layers.append(nn.MaxPool2d(2))
return nn.Sequential(*layers)

class ResNet(ImageClassificationBase):
    def __init__(self, in_channels, num_classes):
        super().__init__()

        self.conv1 = conv_block(in_channels, 128)
        self.conv2 = conv_block(128, 128, pool=True)
        self.res1 = nn.Sequential(conv_block(128, 128), conv_block(128, 128))
        self.drop1 = nn.Dropout(0.5)

        self.conv3 = conv_block(128, 256)
        self.conv4 = conv_block(256, 256, pool=True)
        self.res2 = nn.Sequential(conv_block(256, 256), conv_block(256, 256))
        self.drop2 = nn.Dropout(0.5)

        self.conv5 = conv_block(256, 512)
        self.conv6 = conv_block(512, 512, pool=True)
        self.res3 = nn.Sequential(conv_block(512, 512), conv_block(512, 512))
        self.drop3 = nn.Dropout(0.5)

        self.classifier = nn.Sequential(nn.MaxPool2d(6),
                                        nn.Flatten(),
                                        nn.Linear(512, num_classes))

    def forward(self, xb):
        out = self.conv1(xb)
        out = self.conv2(out)
        out = self.res1(out) + out
        out = self.drop1(out)

        out = self.conv3(out)
        out = self.conv4(out)
        out = self.res2(out) + out
        out = self.drop2(out)

        out = self.conv5(out)
        out = self.conv6(out)
        out = self.res3(out) + out
        out = self.drop3(out)

        out = self.classifier(out)
        return out

The fit_one_cycle function is called to train the model
@torch.no_grad()
def evaluate(model, val_loader):
model.eval()
outputs = [model.validation_step(batch) for batch in val_loader]
return model.validation_epoch_end(outputs)

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
       return param_group['lr']

def fit_one_cycle(epochs, max_lr, model, train_loader, val_loader,
              weight_decay=0, grad_clip=None, opt_func=torch.optim.SGD):
    torch.cuda.empty_cache()
    history = []

# Set up custom optimizer with weight decay
optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)
# Set up one-cycle learning rate scheduler
sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs,
                                            steps_per_epoch=len(train_loader))

    for epoch in range(epochs):
        # Training Phase
        model.train()
        train_losses = []
        lrs = []
        for batch in train_loader:
            loss = model.training_step(batch)
            train_losses.append(loss)
            loss.backward()

            # Gradient clipping
            if grad_clip:
                nn.utils.clip_grad_value_(model.parameters(), grad_clip)

            optimizer.step()
            optimizer.zero_grad()

            # Record & update learning rate
            lrs.append(get_lr(optimizer))
            sched.step()

        # Validation phase
        result = evaluate(model, val_loader)
        result['train_loss'] = torch.stack(train_losses).mean().item()
        result['lrs'] = lrs
        model.epoch_end(epoch, result)
        history.append(result)
    return history

This returns the accuracy and loss, i want to change this so it returns prediction percentages for each class.
    def accuracy(outputs, labels):
    _, preds = torch.max(outputs, dim=1)
    return torch.tensor(torch.sum(preds == labels).item() / len(preds))

class ImageClassificationBase(nn.Module):
    def training_step(self, batch):
        images, labels = batch
        out = self(images)
        loss = F.cross_entropy(out, labels)
        return loss

    def validation_step(self, batch):
        images, labels = batch
        out = self(images)
        loss = F.cross_entropy(out, labels)
        acc = accuracy(out, labels)
        return {'val_loss': loss, 'val_acc': acc}

    def validation_epoch_end(self, outputs):
        batch_losses = [x['val_loss'] for x in outputs]
        epoch_loss = torch.stack(batch_losses).mean()
        batch_accs = [x['val_acc'] for x in outputs]
        epoch_acc = torch.stack(batch_accs).mean()
        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}

    def epoch_end(self, epoch, result):
        print("Epoch [{}], last_lr: {:.5f}, train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}".format(
            epoch, result['lrs'][-1], result['train_loss'], result['val_loss'], result['val_acc']))

Output:
Evaluating the model works, but I can't seem to find how to make a
prediction with a single image. How can I do that?

Simply, if you have a single image make sure to:

use additional 1 dimension at the beginning
make sure to use CHW format instead of HWC (or specify that within pytorch, check out how to do that here)

For example:
my_model = CNN(...)
random_image = torch.randn(1, 3, 100, 100) # 3 channels, 100x100 img

BTW. Your accuracy could be written a little simpler like this:
def accuracy(outputs, labels):
    preds = torch.argmax(outputs, dim=1)
    return torch.sum(preds == labels) / len(preds)

Getting class probability
Similar to argmax you can use softmax which transforms logits (unnormalized probability outputted by your network) into probabilities:
def probability(outputs):
    return torch.nn.functional.softmax(outputs, dim=1)


Instruction: "RuntimeError: mat1 dim 1 must match mat2 dim 0" PyTorch

Input:
I've been trying to build a simple neural network with linear layers:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.lin1 = nn.Linear(64 * 1 * 28 * 28, 6)
        self.lin2 = nn.Linear(6, 4)
        self.out = nn.Linear(4, 10)

    def forward(self, x):
        print(x.shape)

        x = self.lin1(x)
        x = F.relu(x)

        x = self.lin2(x)
        x = F.relu(x)

        x = self.out(x)
        x = F.softmax(x)

        return x

However I run into the error:

mat1 dim 1 must match mat2 dim 0

for the line
x = self.lin1(x)

I tried flattening x before lin1 or changing the lin1 input size, but nothing worked.
The output of the print statement is:
torch.Size([64, 1, 28, 28])

Output:
Problem was solved by flattening (in a different way than i did before) the input.
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
  def __init__(self):
    super(Net,self).__init__()

    self.lin1 = nn.Linear(784,6)
    self.lin2  = nn.Linear(6,4)
    self.out = nn.Linear(4,10)

def forward(self,x):

    x = torch.flatten(x,start_dim = 1) #This is new

    x = self.lin1(x)
    x = F.relu(x)

    x = self.lin2(x)
    x = F.relu(x)

    x = self.out(x)

    return x


Instruction: Huggingface Electra - Load model trained with google implementation error: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte

Input:
I have trained an electra model from scratch using google implementation code.
python run_pretraining.py --data-dir gc://bucket-electra/dataset/ --model-name greek_electra --hparams hparams.json

with this json hyperparams:
{
"embedding_size": 768,
"max_seq_length": 512,
"train_batch_size": 128,
"vocab_size": 100000,
"model_size": "base",
"num_train_steps": 1500000
}

After having trained the model, I used the convert_electra_original_tf_checkpoint_to_pytorch.py script from transformers library to convert the checkpoint.
python convert_electra_original_tf_checkpoint_to_pytorch.py --tf_checkpoint_path output/models/transformer/greek_electra --config_file resources/hparams.json --pytorch_dump_path output/models/transformer/discriminator  --discriminator_or_generator "discriminator"

Now I am trying to load the model:
from transformers import ElectraForPreTraining

model = ElectraForPreTraining.from_pretrained('discriminator')

but I get the following error:
Traceback (most recent call last):
  File "~/.local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 427, in get_config_dict
    config_dict = cls._dict_from_json_file(resolved_config_file)
  File "~/.local/lib/python3.9/site-packages/transformers/configuration_utils.py", line 510, in _dict_from_json_file
    text = reader.read()
  File "/usr/lib/python3.9/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte

Any ideas what's causing this & how to solve it?

Output: It seems that @npit is right. The output of the convert_electra_original_tf_checkpoint_to_pytorch.py does not contain the configuration that I gave (hparams.json), therefore I created an ElectraConfig object -- with the same parameters -- and provided it to the from_pretrained function. That solved the issue.


Instruction: How can/should we weight classes in HuggingFace token classification (entity recognition)?

Input:
I'm training a token classification (AKA named entity recognition) model with the HuggingFace Transformers library, with a customized data loader.
Like most NER datasets (I'd imagine?) there's a pretty significant class imbalance: A large majority of tokens are other - i.e. not an entity - and of course there's a little variation between the different entity classes themselves.
As we might expect, my "accuracy" metrics are getting distorted quite a lot by this: It's no great achievement to get 80% token classification accuracy if 90% of your tokens are other... A trivial model could have done better!
I can calculate some additional and more insightful evaluation metrics - but it got me wondering... Can/should we somehow incorporate these weights into the training loss? How would this be done using a typical *ForTokenClassification model e.g. BERTForTokenClassification?

Output:
This is actually a really interesting question, since it seems there is no intention (yet) to modify losses in the models yourself. Specifically for BertForTokenClassification, I found this code segment:
loss_fct = CrossEntropyLoss()
# ...
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))

To actually change the loss computation and add other parameters, e.g., the weights you mention, you can go about either one of two ways:

You can modify a copy of transformers locally, and install the library from there, which makes this only a small change in the code, but potentially quite a hassle to change parts during different experiments, or
You return your logits (which is the case by default), and calculate your own loss outside of the actual forward pass of the huggingface model. In this case, you need to be aware of any potential propagation from the loss calculated within the forward call, but this should be within your power to change.


Instruction: set variable network layers based on parameters in pytorch

Input:
I want to make the following network definition to a parametric one. The number of continuous and discrete columns varies for different data. I first pass the whole input data, which in this case is 110 dimensional , from a linear with a relu activation. The output of each categorical field of my data varies based on a previous one-hot encoding data transformation. I need to define a nn.Linear(110, number of encodings) for each of them.
class Generator(nn.Module):
  def __init__(self):
    super(Generator, self).__init__(110)
    self.lin1 = nn.Linear(110,110)
    self.lin_numerical = nn.Linear(110, 6)
    self.lin_cat_job = nn.Linear(110, 9)
    self.lin_cat_sex = nn.Linear(110, 2)
    self.lin_cat_incomeclass = nn.Linear(110, 7)

  def forward(self, x):
    x = torch.relu(self.lin1(x))
    x_numerical = f.leaky_relu(self.lin_numerical(x))

    x_cat1 = f.gumbel_softmax(self.lin_cat_job(x), tau=0.2)
    x_cat2 = f.gumbel_softmax(self.lin_cat_sex(x), tau=0.2)
    x_cat3 = f.gumbel_softmax(self.lin_cat_incomeclass(x), tau=0.2)

    x_final = torch.cat((x_numerical, x_cat1, x_cat2, x_cat3),1)
    return x_final

I have managed to change the init part, using discrete_columns input which is an ordereddict that has the name and number of one-hot-encoding of each categorical field of my data as key and values, and continuous_columns which is only a list with the names of the continuous columns. But I have no idea how to edit the forward part:
class Generator(nn.Module):
  def __init__(self, input_dim, continuous_columns, discrete_columns):
    super(Generator, self).__init__()
    self._input_dim = input_dim
    self._discrete_columns = discrete_columns
    self._num_continuous_columns = len(continuous_columns)

    self.lin1 = nn.Linear(self._input_dim, self._input_dim)
    self.lin_numerical = nn.Linear(self._input_dim, self._num_continuous_columns)

    for key, value in self._discrete_columns.items():
      setattr(self, "lin_cat_{}".format(key), nn.Linear(self._input_dim, value))

  def forward(self, x):
    x = torch.relu(self.lin1(x))
    x_numerical = f.leaky_relu(self.lin_numerical(x))
    ####
    This is the problematic part
    #####
    return x

Output:
You don't need to use setattr and honestly should not since you'd need getattr, it brings more trouble than it solves if there's any other ways to do the job.
Now this is what I'd do for this task
        self.lin_cat = nn.ModuleDict()
        for key, value in self._discrete_columns.items():
            self.lin_cat[key] = nn.Linear(self._input_dim, value)
        #     setattr(self, "lin_cat_{}".format(key), nn.Linear(self._input_dim, value))

    def forward(self, x):
        x = torch.relu(self.lin1(x))
        x_numerical = f.leaky_relu(self.lin_numerical(x))

        x_cat = []
        for key in self.lin_cat:
            x_cat.append(f.gumbel_softmax(self.lin_cat[key](x), tau=0.2))
        x_final = torch.cat((x_numerical, *x_cat), 1)
        return x


Instruction: Setting a minimum learning rate on "Reduce On Plateau"

Input:
I'm currently using PyTorch's ReduceLROnPlateau learning rate scheduler using:
learning_rate = 1e-3
optimizer = optim.Adam(model.params, lr = learning_rate)
model.optimizer = optimizer
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(model.optimizer, factor=0.9, patience = 5000, verbose=True)

My issue is that the loss will get really low within a few minutes, then jumps really high and start decreasing steadily. Hence, ReduceLROnPlateau will just continuously decrease the learning rate to a point that the model can't learn anything (because it takes the lowest loss into acount).
Is there a way to set a minimum learning rate using this scheduler so that it won't go to almost 0? Say to set it at minimum of 1e8

Output:
Yes, from the documentation:

min_lr (float or list) – A scalar or a list of scalars. A lower bound on the learning rate of all param groups or each group respectively. Default: 0.

You can simply go for:
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
    model.optimizer,
    factor=0.9,
    patience=5000,
    verbose=True,
    min_lr=1e-8,
)


Instruction: torch.Tensor() new() received an invalid combination of arguments - got (list, dtype=torch.dtype)

Input:
I am facing a very weird error when I am trying to run the following simple line :
a = torch.Tensor([0,0,0],dtype = torch.int64)

TypeError: new() received an invalid combination of arguments - got (list, dtype=torch.dtype), but expected one of:
 * (*, torch.device device)
      didn't match because some of the keywords were incorrect: dtype
 * (torch.Storage storage)
 * (Tensor other)
 * (tuple of ints size, *, torch.device device)
 * (object data, *, torch.device device)

whereas if we look at official documentation
torch.tensor(data, dtype=None, device=None, requires_grad=False, pin_memory=False) → Tensor

Parameters
data (array_like) – Initial data for the tensor. Can be a
list, tuple, NumPy ndarray, scalar, and other types.
dtype (torch.dtype, optional) – the desired data type of returned
tensor. Default: if None, infers data type from data.

Why is the code snippet not working even though the official documentation supports it?

Output: Capitalization matters -- in your top example, you have Tensor with uppercase T, but the documentation excerpt is talking about tensor with lowercase t.


Instruction: Can’t train a multilabel classifier on a small dataset

Input:
I have a dataset of 3 classes animals, landscapes, and buildings. It is only a 100 images per dataset and I am trying to train a classifier using ResNet34 + fastai but I'm running into several issues.
The first one is that I don't think my model is training properly. When I use the lr_finder, my validation loss is na, unless this is expected:

Then I run 10 epochs. It looks either like its learning well or its overfitting like hell.

Then I unfreeze to find another learning rate and my validation loss is still na:

And now when I train 10 epochs, the graph starts to oscillate so that tells me that it can't learn anymore. The accuracy looks good (unless its overfitting) so then I decide to use my validation set to see how my model is actually doing:

Because I have 300 images, 240 are in the train folder and 60 are in the test folder. My test images are labelled because I want to run an accuracy score on it. This is the cell that I run:
    path = '/content/dataset/'

    data_test = ImageList.from_folder(path).split_by_folder(train='train', valid='test').label_from_re(file_parse).transform(size=512).databunch().normalize(imagenet_stats)

    learn = cnn_learner(data, models.resnet50, metrics=[accuracy, top_1],callback_fns=ShowGraph)

    learn.load('stage-2')

And this is the error that I am met with:
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-28-f1232eb8cf47> in <module>()
      4
      5 learn = cnn_learner(data, models.resnet50, metrics=[accuracy, top_1],callback_fns=ShowGraph)
    ----> 6 learn.load('stage-2')

    1 frames
    /usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in load_state_dict(self, state_dict, strict)
    828         if len(error_msgs) > 0:
    829             raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
    --> 830                                self.__class__.__name__, "\n\t".join(error_msgs)))
    831         return _IncompatibleKeys(missing_keys, unexpected_keys)
    832

    RuntimeError: Error(s) in loading state_dict for Sequential:
        Missing key(s) in state_dict: "0.4.0.conv3.weight", "0.4.0.bn3.weight", "0.4.0.bn3.bias", "0.4.0.bn3.running_mean", "0.4.0.bn3.running_var", "0.4.0.downsample.0.weight", "0.4.0.downsample.1.weight", "0.4.0.downsample.1.bias", "0.4.0.downsample.1.running_mean", "0.4.0.downsample.1.running_var", "0.4.1.conv3.weight", "0.4.1.bn3.weight", "0.4.1.bn3.bias", "0.4.1.bn3.running_mean", "0.4.1.bn3.running_var", "0.4.2.conv3.weight", "0.4.2.bn3.weight", "0.4.2.bn3.bias", "0.4.2.bn3.running_mean", "0.4.2.bn3.running_var", "0.5.0.conv3.weight", "0.5.0.bn3.weight", "0.5.0.bn3.bias", "0.5.0.bn3.running_mean", "0.5.0.bn3.running_var", "0.5.1.conv3.weight", "0.5.1.bn3.weight", "0.5.1.bn3.bias", "0.5.1.bn3.running_mean", "0.5.1.bn3.running_var", "0.5.2.conv3.weight", "0.5.2.bn3.weight", "0.5.2.bn3.bias", "0.5.2.bn3.running_mean", "0.5.2.bn3.running_var", "0.5.3.conv3.weight", "0.5.3.bn3.weight", "0.5.3.bn3.bias", "0.5.3.bn3.running_mean", "0.5.3.bn3.running_var", "0.6.0.conv3.weight", "0.6.0.bn3.weight", "0.6.0.bn3.bias", "0.6.0.bn3.running_mean", "0.6.0.bn3.running_var", "0.6.1.conv3.weight", "0.6.1.bn3.weight", "0.6.1.bn3.bias", "0.6.1.bn3.running_mean", "0.6.1.bn3.running_var", "0.6.2.conv3.weight", "0.6.2.bn3.weight", "0.6.2.bn3.bias", "0.6.2.bn3.running_mean", "0.6.2.bn3.running_var", "0.6.3.conv3.weight", "0.6.3.bn3.weight", "0.6.3.bn3.bias", "0.6.3.bn3.running_mean", "0.6.3.bn3.running_var", "0.6.4.conv3.weight", "0.6.4.bn3.weight", "0.6.4.bn3.bias", "0.6.4.bn3.running_mean", "0.6....
        size mismatch for 0.4.0.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 64, 1, 1]).
        size mismatch for 0.4.1.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 256, 1, 1]).
        size mismatch for 0.4.2.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 256, 1, 1]).
        size mismatch for 0.5.0.conv1.weight: copying a param with shape torch.Size([128, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 256, 1, 1]).
        size mismatch for 0.5.0.downsample.0.weight: copying a param with shape torch.Size([128, 64, 1, 1]) from checkpoint, the shape in current model is torch.Size([512, 256, 1, 1]).
        size mismatch for 0.5.0.downsample.1.weight: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.bias: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.running_mean: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.running_var: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.1.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.5.2.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.5.3.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.6.0.conv1.weight: copying a param with shape torch.Size([256, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 512, 1, 1]).
        size mismatch for 0.6.0.downsample.0.weight: copying a param with shape torch.Size([256, 128, 1, 1]) from checkpoint, the shape in current model is torch.Size([1024, 512, 1, 1]).
        size mismatch for 0.6.0.downsample.1.weight: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.running_mean: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.running_var: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.1.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.2.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.3.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.4.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.5.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.7.0.conv1.weight: copying a param with shape torch.Size([512, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 1024, 1, 1]).
        size mismatch for 0.7.0.downsample.0.weight: copying a param with shape torch.Size([512, 256, 1, 1]) from checkpoint, the shape in current model is torch.Size([2048, 1024, 1, 1]).
        size mismatch for 0.7.0.downsample.1.weight: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.bias: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.running_mean: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.running_var: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.1.conv1.weight: copying a param with shape torch.Size([512, 512, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 2048, 1, 1]).
        size mismatch for 0.7.2.conv1.weight: copying a param with shape torch.Size([512, 512, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 2048, 1, 1]).
        size mismatch for 1.2.weight: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.bias: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.running_mean: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.running_var: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.4.weight: copying a param with shape torch.Size([512, 1024]) from checkpoint, the shape in current model is torch.Size([512, 4096]).

Output:
There are few issues with your approach  :

You mentioned that you used resnet 34 for training and then saving its weights. Yet while testing, you are using resnet 50 and trying to load the weights of resnet 34 which will not work due to different arch (layers & parameters).

In case  you are doing the testing in the same NB , you cold have added test dataset  while creating the train & valid dataset itself and get predictions on the whole dataset with 2 lines of code. a quick sample :
test_imgs = (path/'cars_test/').ls()
data.add_test(test_imgs)
learn.data = data
preds = learn.get_preds(ds_type=DatasetType.Test)

If you are going to use it in different nb , try model.export to save model, its weights, data and all the info . I think jeremy did that in his NB and course.


Instruction: Difference between Tensor and tensor

Input:
I am working with Pytorch. But when I use tensor I got an error while I don't get any error when I use Tensor.
What is the difference between Tensor and tensor?
This is my example:
tns = torch.tensor([91,21,34,56])

tns.mean()

I got this error:

RuntimeError: Can only calculate the mean of floating types. Got Long
instead.

Thanks in advance.

Output:
If you don't specify a dtype in torch.tensor() it infers this from the data. Since your data is all integers, it used Long.
torch.mean() can only be calculated for floats.
By default, torch.Tensor is an alias for torch.FloatTensor hence it automatically has dtype float (unless you change the default behaviour).


Instruction: Is reshape() same as contiguous().view()?

Input: In PyTorch, for a tensor x is x.reshape(shape) always equivalent to x.contiguous().view(shape)?

Output:
No. There are some circumstances where .reshape(shape) can create a view, but .contiguous().view(shape) will create a copy.
Here is an example:
x = torch.zeros(8, 10)
y = x[:, ::2]
z0 = y.reshape(40)               # Makes a new view
z1 = y.contiguous().view(40)     # Makes a copy

We can confirm that z0 is a new view of x, but z1 is a copy:
> x.data_ptr() == z0.data_ptr()
True
> x.data_ptr() == z1.data_ptr()
False


Instruction: Torch argmax for N highest values

Input:
I would like to do something like an argmax but with multiple top values. I know how to use the normal torch.argmax
>>> a = torch.randn(4, 4)
>>> a
tensor([[ 1.3398,  1.2663, -0.2686,  0.2450],
        [-0.7401, -0.8805, -0.3402, -1.1936],
        [ 0.4907, -1.3948, -1.0691, -0.3132],
        [-1.6092,  0.5419, -0.2993,  0.3195]])
>>> torch.argmax(a)
tensor(0)

But now I need to find the indices for the top N values. So something like this
>>> a = torch.randn(4, 4)
>>> a
tensor([[ 1.3398,  1.2663, -0.2686,  0.2450],
        [-0.7401, -0.8805, -0.3402, -1.1936],
        [ 0.4907, -1.3948, -1.0691, -0.3132],
        [-1.6092,  0.5419, -0.2993,  0.3195]])
>>> torch.argmax(a,top_n=2)
tensor([0,1])

I haven't found any function capable of doing this in pytorch, does anyone know?

Output:
Great! So you need the first k largest elements of a tensor.
[Answer 1] You need the first k largest of all the elements irrespective of the dimension. So, flatten the tensor and use the torch.topk function to get indices of top-3 (for example) elements:
>>> a = torch.randn(5,4)
>>> a
tensor([[ 0.8292, -0.5123, -0.0741, -0.3043],
        [-0.4340, -0.7763,  1.9716, -0.5620],
        [ 0.1582, -1.2000,  1.0202, -1.5202],
        [-0.3617, -0.2479,  0.6204,  0.2575],
        [ 1.8025,  1.9864, -0.8013, -0.7508]])
>>> torch.topk(a.flatten(), 3).indices
tensor([17,  6, 16])

[Answer 2] You need k largest elements of the given input tensor along a given dimension. So for this refer to the PyTorch documentation of the function torch.topk given here.


Instruction: Batchnormalization over which dimension?

Input:
Over which dimension do we calculate the mean and std? Is it over the hidden dimensions of the NN Layer, or over all the samples in the batch for every hidden dimension separately?
In the paper it says we normalize over the batch.
In torch.nn.BatchNorm1d however the input argument is num_features, which makes no sense to me.
Why does pytorch not follow the original paper on Batchnormalization?

Output:
over which dimension do we calculate the mean and std?

Over 0th dimension, for 1D input of shape (batch, num_features) it would be:
batch = 64
features = 12
data = torch.randn(batch, features)

mean = torch.mean(data, dim=0)
var = torch.var(data, dim=0)

In torch.nn.BatchNorm1d hower the input argument is "num_features",
which makes no sense to me.

It is not related to normalization but reparametrization of mean and var via gamma and beta learnable parameters. From the paper:

Both parameters used in scale and shift phase are of shape num_features, hence we have to pass this value in order to initialize them with specific shape.
Below is an example from scratch implementation for reference:
class BatchNorm1d(torch.nn.Module):
    def __init__(self, num_features, momentum: float = 0.9, eps: float = 1e-7):
        super().__init__()
        self.num_features = num_features

        self.gamma = torch.nn.Parameter(torch.ones(1, self.num_features))
        self.beta = torch.nn.Parameter(torch.zeros(1, self.num_features))

        self.register_buffer("running_mean", torch.ones(1, self.num_features))
        self.register_buffer("running_var", torch.ones(1, self.num_features))

        self.momentum = momentum
        self.eps = eps

    def forward(self, X):
        if not self.training:
            X_hat = X - self.running_mean / torch.sqrt(self.running_var + self.eps)
        else:
            mean = X.mean(dim=0).unsqueeze(dim=0)
            var = ((X - mean) ** 2).mean(dim=0).unsqueeze(dim=0)

            # Update running mean and variance
            self.running_mean *= self.momentum
            self.running_mean += (1 - self.momentum) * mean

            self.running_var *= self.momentum
            self.running_var += (1 - self.momentum) * var

            X_hat = X - mean / torch.sqrt(var + self.eps)

        return X_hat * self.gamma + self.beta

Why does pytorch not follow the original paper on Batchnormalization?

It does as one can see


Instruction: How to correctly set input image size for LPRNet model? 'RuntimeError: Sizes of tensors must match except in dimension 1..'

Input:
Trying to adapt the LPRNet model for my own dataset. The original model was set for Chinese license plate images with dimensions 24x94 pixels. My dataset consists of plates with only numbers, images come with dimensions 64x128. I got an error when I try to replace 'summary(lprnet, (3,24,94), device="cpu")' to 'summary(lprnet, (3,64,128), device="cpu")'.
The first code is working well, the second gets 'RuntimeError: Sizes of tensors must match except in dimension 1. Got 25 and 24 in dimension 3 (The offending index is 1)', this is the last line of code.
I don't see where else in the code I should change parameters. Will be thankful for any clue!
Original: https://github.com/xuexingyu24/License_Plate_Detection_Pytorch/blob/master/LPRNet/model/LPRNET.py
# LPRNET model
import torch
import torch.nn as nn
from torchsummary import summary

class small_basic_block(nn.Module):
    def __init__(self, ch_in, ch_out):
        super(small_basic_block, self).__init__()
        self.block = nn.Sequential(
            nn.Conv2d(ch_in, ch_out // 4, kernel_size=1),
            nn.ReLU(),
            nn.Conv2d(ch_out // 4, ch_out // 4, kernel_size=(3, 1), padding=(1, 0)),
            nn.ReLU(),
            nn.Conv2d(ch_out // 4, ch_out // 4, kernel_size=(1, 3), padding=(0, 1)),
            nn.ReLU(),
            nn.Conv2d(ch_out // 4, ch_out, kernel_size=1),
        )
    def forward(self, x):
        return self.block(x)

class LPRNet(nn.Module):
    def __init__(self, class_num, dropout_rate):
        super(LPRNet, self).__init__()
        self.class_num = class_num
        self.backbone = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=3, stride=1), # 0
            nn.BatchNorm2d(num_features=64),
            nn.ReLU(),  # 2
            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 1, 1)),
            small_basic_block(ch_in=64, ch_out=128),    # *** 4 ***
            nn.BatchNorm2d(num_features=128),
            nn.ReLU(),  # 6
            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(2, 1, 2)),
            small_basic_block(ch_in=64, ch_out=256),   # 8
            nn.BatchNorm2d(num_features=256),
            nn.ReLU(),  # 10
            small_basic_block(ch_in=256, ch_out=256),   # *** 11 ***
            nn.BatchNorm2d(num_features=256),   # 12
            nn.ReLU(),
            nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(4, 1, 2)),  # 14
            nn.Dropout(dropout_rate),
            nn.Conv2d(in_channels=64, out_channels=256, kernel_size=(1, 4), stride=1),  # 16
            nn.BatchNorm2d(num_features=256),
            nn.ReLU(),  # 18
            nn.Dropout(dropout_rate),
            nn.Conv2d(in_channels=256, out_channels=class_num, kernel_size=(13, 1), stride=1), # 20
            nn.BatchNorm2d(num_features=class_num),
            nn.ReLU(),  # *** 22 ***
        )
        self.container = nn.Sequential(
            nn.Conv2d(in_channels=256+class_num+128+64, out_channels=self.class_num, kernel_size=(1,1), stride=(1,1)),
            # nn.BatchNorm2d(num_features=self.class_num),
            # nn.ReLU(),
            # nn.Conv2d(in_channels=self.class_num, out_channels=self.lpr_max_len+1, kernel_size=3, stride=2),
            # nn.ReLU(),
        )

    def forward(self, x):
        keep_features = list()
        for i, layer in enumerate(self.backbone.children()):
            x = layer(x)
            if i in [2, 6, 13, 22]: # [2, 4, 8, 11, 22]
                keep_features.append(x)

        global_context = list()
        for i, f in enumerate(keep_features):
            if i in [0, 1]:
                f = nn.AvgPool2d(kernel_size=5, stride=5)(f)
            if i in [2]:
                f = nn.AvgPool2d(kernel_size=(4, 10), stride=(4, 2))(f)
            f_pow = torch.pow(f, 2)
            f_mean = torch.mean(f_pow)
            f = torch.div(f, f_mean)
            global_context.append(f)

        x = torch.cat(global_context, 1)
        x = self.container(x)
        logits = torch.mean(x, dim=2)

        return logits

CHARS = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

if __name__ == "__main__":

    lprnet = LPRNet(class_num=len(CHARS), dropout_rate=0)
    print(lprnet)

    summary(lprnet, (3,24,94), device="cpu")
    #summary(lprnet, (3,64,128), device="cpu")

Output:
The problem with your code is that the shapes of four tensors in global_context are different for (64, 128) input size. For shape (24, 94), in LRPNet, the authors make all the tensors of (4, 18) sized with average pooling which doesn't apply to your image size. To make them same-sized, you need to change the configurations of nn.AvgPool2d while enumerating keep_features. You may update them in various ways. I updated the avgpool with the below configurations and it worked:
            if i in [0, 1]:
                f = nn.AvgPool2d(kernel_size=(11, 24), stride=(7, 4))(f)

            elif i == 2:
                f = nn.AvgPool2d(kernel_size=(9, 11), stride=(7, 2))(f)
            elif i == 3:
                f = nn.AvgPool2d(kernel_size=(7, 2), stride=(5, 1))(f)
            f_pow = torch.pow(f, 2)
            f_mean = torch.mean(f_pow)
            f = torch.div(f, f_mean)
            global_context.append(f)```


Instruction: How to load a saved model defined in a function in PyTorch in Colab?

Input:
Here is a sample code of my training function(unnecessary parts are deleted):
I am trying to save my model data_gen in the torch.save(), and after running the train_dmc function, I can find the checkpoint file in the directory.
def train_dmc(loader,loss):

  data_gen = DataGenerator().to(device)

  data_gen_optimizer = optim.Rprop(para_list, lr=lrate)

  savepath='/content/drive/MyDrive/'+loss+'checkpoint.t7'
  state = {
            'epoch': epoch,
            'model_state_dict': data_gen.state_dict(),
            'optimizer_state_dict': data_gen_optimizer.state_dict(),
            'data loss': data_loss,
            'latent_loss':latent_loss
            }
  torch.save(state,savepath)

My question is that how to load the checkpoint file to continue training if Google Colab disconnects.
Should I load data_gen or train_dmc(), it is my first time using this and I am really confused because the data_gen is defined inside another function. Hope someone can help me with explanation
data_gen.load_state_dict(torch.load(PATH))
data_gen.eval()

#or

train_dmc.load_state_dict(torch.load(PATH))
train_dmc.eval()

Output:
As the state variable is a dictionary, So try saving it as:
with open('/content/checkpoint.t7', 'wb') as handle:
    pickle.dump(state, handle, protocol=pickle.HIGHEST_PROTOCOL)

Initiate your model class as data_gen = DataGenerator().to(device).
And load the checkpoint file as:
import pickle
file = open('/content/checkpoint.t7', 'rb')
loaded_state = pickle.load(file)

Then you can load the state_dict using data_gen = loaded_state['model_state_dict']. This will load the state_dict to the model class!


Instruction: How to use MSELoss function for Fashion_MNIST in pytorch?

Input:
I want to get through Fashion_Mnist data, I would like to see the output gradient which might be mean squared sum between first and second layer
My code first below
#import the nescessary libs
import numpy as np
import torch
import time

# Loading the Fashion-MNIST dataset
from torchvision import datasets, transforms

# Get GPU Device

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.cuda.get_device_name(0)

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize((0.5,), (0.5,))
                                                                   ])
# Download and load the training data
trainset = datasets.FashionMNIST('MNIST_data/', download = True, train = True, transform = transform)
testset = datasets.FashionMNIST('MNIST_data/', download = True, train = False, transform = transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size = 128, shuffle = True, num_workers=4)
testloader = torch.utils.data.DataLoader(testset, batch_size = 128, shuffle = True, num_workers=4)

# Examine a sample
dataiter = iter(trainloader)
images, labels = dataiter.next()

# Define the network architecture
from torch import nn, optim
import torch.nn.functional as F

model = nn.Sequential(nn.Linear(784, 128),
                      nn.ReLU(),
                      nn.Linear(128, 10),
                      nn.LogSoftmax(dim = 1)
                     )
model.to(device)

# Define the loss
criterion = nn.MSELoss()

# Define the optimizer
optimizer = optim.Adam(model.parameters(), lr = 0.001)

# Define the epochs
epochs = 5
train_losses, test_losses = [], []
squared_sum = []
# start = time.time()
for e in range(epochs):
    running_loss = 0

    for images, labels in trainloader:
    # Flatten Fashion-MNIST images into a 784 long vector
        images = images.to(device)
        labels = labels.to(device)
        images = images.view(images.shape[0], -1)

        optimizer.zero_grad()

        output = model[0].forward(images)
        loss = criterion(output[0], labels.float())

        loss.backward()

        optimizer.step()
        running_loss += loss.item()

    else:

        print(running_loss)
        test_loss = 0
        accuracy = 0

    # Turn off gradients for validation, saves memory and computation
        with torch.no_grad():
      # Set the model to evaluation mode
            model.eval()

      # Validation pass
            for images, labels in testloader:
                images = images.to(device)
                labels = labels.to(device)
                images = images.view(images.shape[0], -1)
                ps = model(images[0])
                test_loss += criterion(ps, labels)
                top_p, top_class = ps.topk(1, dim = 1)
                equals = top_class == labels.view(*top_class.shape)
                accuracy += torch.mean(equals.type(torch.FloatTensor))

    model.train()
    print("Epoch: {}/{}..".format(e+1, epochs),
          "Training loss: {:.3f}..".format(running_loss/len(trainloader)),
          "Test loss: {:.3f}..".format(test_loss/len(testloader)),
          "Test Accuracy: {:.3f}".format(accuracy/len(testloader)))

What I want to get,
for e in range(epochs):
    running_loss = 0

    for images, labels in trainloader:
    # Flatten Fashion-MNIST images into a 784 long vector
        images = images.to(device)
        labels = labels.to(device)
        images = images.view(images.shape[0], -1)

        optimizer.zero_grad()

        output = model[0].forward(images)
        loss = criterion(output[0], labels.float())

        loss.backward()

        optimizer.step()
        running_loss += loss.item()

In here, model[0] (This might be the first layer nn.Linear(784, 128)), I would love to get the mean square errors only for first and second layer,
If I run this code, I receive this error below
RuntimeError: The size of tensor a (128) must match the size of tensor b (96) at non-singleton dimension 0

If I want to run this code correctly to get the MSELoss, what I need to do?

Output:
The error is caused by the number of samples in the dataset and the batch size.
In more detail, the training MNIST dataset includes 60,000 samples, your current batch_size is 128 and you will need 60000/128=468.75 loops to finish training on one epoch. So the problem comes from here, for 468 loops, your data will have 128 samples but the last loop just contains 60000 - 468*128 = 96 samples.
To solve this problem, I think you need to find the suitable batch_size and the number of neural in your model as well.
I think it should work for computing loss
trainloader = torch.utils.data.DataLoader(trainset, batch_size = 96, shuffle = True, num_workers=0)
testloader = torch.utils.data.DataLoader(testset, batch_size = 96, shuffle = True, num_workers=0)
model = nn.Sequential(nn.Linear(784, 96),
                      nn.ReLU(),
                      nn.Linear(96, 10),
                      nn.LogSoftmax(dim = 1)
                     )


Instruction: Importing fastai in GoogleColab

Input:
I am trying to import this module in googlecolab for my code. As I want to run my code with GPU but even when I do pip install fastai or pip3 intall fastai , it says requirement satisfied but gives me error when I run these:
from fastai.vision.all import *
import gc

ModuleNotFoundError: No module named 'fastai.vision.all'

Output:
You need to upgrade fastai. Run the following instead:
!pip install fastai --upgrade

Then, check if the installed version is 2.0 or higher:
import fastai

print(fastai.__version__)


Instruction: Apply stochastic depth by Pytorch

Input:
The definition of Stochastic depth was first mentioned in this paper. In short, it's similar to drop-out but instead of node, it will terminate the connection of the Skip connection structure (residual block) in ResNet paper.
My question: is there any fast, easy way to implement Stochastic depth in transfer learning by Pytorch such as drop-out (simply add torch.nn.DropOut(p) into the classifier block).

Output:
Basic StochasticDepth
Yes, one could do something like this pretty easily:
class StochasticDepth(torch.nn.Module):
    def __init__(self, module: torch.nn.Module, p: float = 0.5):
        super().__init__()
        if not 0 < p < 1:
            raise ValueError(
                "Stochastic Depth p has to be between 0 and 1 but got {}".format(p)
            )
        self.module: torch.nn.Module = module
        self.p: float = p
        self._sampler = torch.Tensor(1)

    def forward(self, inputs):
        if self.training and self._sampler.uniform_():
            return inputs
        return self.p * self.module(inputs)

Please notice that:

inputs shape has to be the same as self.module(inputs) shape
You can pass any block inside this function (see below)

Example usage:
layer = StochasticDepth(
    torch.nn.Sequential(
        torch.nn.Linear(10, 10),
        torch.nn.ReLU(),
        torch.nn.Linear(10, 10),
        torch.nn.ReLU(),
    ),
    p=0.5,
)

Adding to existing models
First, you should print the model that you want and analyze the weights and outputs.
What you're looking for to apply this module easiest (in case of Conv{1,2,3}d layers):

same number of in_channels and out_channels within the block
for different number of in_channels and out_channels some kind of projection would be needed

StochasticDepth with projection
Version with projection of StochasticDepth:
class StochasticDepth(torch.nn.Module):
    def __init__(
        self,
        module: torch.nn.Module,
        p: float = 0.5,
        projection: torch.nn.Module = None,
    ):
        super().__init__()
        if not 0 < p < 1:
            raise ValueError(
                "Stochastic Depth p has to be between 0 and 1 but got {}".format(p)
            )
        self.module: torch.nn.Module = module
        self.p: float = p
        self.projection: torch.nn.Module = projection
        self._sampler = torch.Tensor(1)

    def forward(self, inputs):
        if self.training and self._sampler.uniform_():
            if self.projection is not None:
                return self.projection(inputs)
            return inputs
        return self.p * self.module(inputs)

projection could be Conv2d(256, 512, kernel_size=1, stride=2) in case of resnet modules as it would increase number of channels and make the image smaller via stride=2 as in original paper.
Applying StochasticDepth
If you print torchvision.models.resnet18() you would see repeating blocks like this:
(layer2): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (downsample): Sequential(
        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (relu): ReLU(inplace=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
  )

Each layer is a larger colored block you may want to randomly skip. For resnet18 and layer specifically one could do this:
model = torchvision.models.resnet18()
model.layer1 = StochasticDepth(model.layer1)
model.layer2 = StochasticDepth(
    model.layer2,
    projection=torch.nn.Conv2d(
        64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False
    ),
)
model.layer3 = StochasticDepth(
    model.layer3,
    projection=torch.nn.Conv2d(
        128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False
    ),
)
model.layer4 = StochasticDepth(
    model.layer4,
    projection=torch.nn.Conv2d(
        256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False
    ),
)

First block's channels stay the same hence no projection is needed
Second, third and fourth up number of channels and make the image smaller via stride hence simple projection is used

You can modify any part of the neural network using this approach, just remember to test whether the shapes agree.
Simpler projections
One could also tie weights between first conv layer in specific block and use that module as a projection, see below:
model = torchvision.models.resnet18()
model.layer1 = StochasticDepth(model.layer1)
model.layer2 = StochasticDepth(model.layer2, projection=model.layer2[0].conv1)
model.layer3 = StochasticDepth(model.layer3, projection=model.layer3[0].conv1)
model.layer4 = StochasticDepth(model.layer4, projection=model.layer4[0].conv1)

Upsides:

weights are not randomly initialized
easier to write

Downsides:

weights are tied and one layer will have to do two tasks:

be first in the block (without dropping)
be the only in the block (with dropping)

this might not end up too well as it is responsible for conflicting tasks

You may also copy this module instead of sharing weights, probably best idea.


Instruction: Pytorch: Creating a tensor mask based of rate/percentage/ratio

Input:
How can I generate a mask tensor that has a specific ratio of 0 and 1? For example 70:30 of 0s and 1s in a 5 by 10 tensor will generate:
[[0,0,0,0,0,0,0,0,1,0],
[1,1,1,0,0,0,1,1,0,0],
[0,0,1,0,0,1,1,0,1,0],
[0,1,0,1,0,1,1,0,0,0],
[0,0,0,1,0,0,0,0,0,0]]

Output:
You can approximate the ratio using random sampling:
import torch

mask = torch.rand(5, 10)  # uniformly distributed between 0 and 1
mask = mask < 0.3   # 30% pixels "on"

On average, mask will have the right amount of "on" pixels.

Alternatively, if you must have exactly 30% of "on" pixels, you can use torch.randperm to randomly permute a mask with exactly the number of "on" pixels:
raw = torch.zeros((5*10,))
raw[:int(0.3 * 5 * 10)] = 1.  # set EXACTLY 30% of the pixels in the mask
ridx = torch.randperm(5*10)   # a random permutation of the entries
mask = torch.reshape(raw[ridx], (5, 10))


Instruction: How can I generate lots of FC layer in a simple way?

Input:
I would like to make 640 fc layer.
(in def init)
self.fc0 = nn.Linear(120, M)
self.fc1 = nn.Linear(120, M)
.....
self.fc638 = nn.Linear(120, M)
self.fc639 = nn.Linear(120, M)

(in def forward)
x[:,:,0,:] = self.fc0(x[:,:,0,:])
x[:,:,1,:] = self.fc0(x[:,:,1,:])
.......
x[:,:,639,:] = self.fc639(x[:,:,639,:])

How can I execute the code above in simpler way ?

Output:
Use containers:
class MultipleFC(nn.Module):
  def __init__(self, num_layers, M):
    self.layers = nn.ModuleList([nn.Linear(120, M) for _ in range(num_layers)])

  def forward(self, x):
    y = torch.empty_like(x)  # up to third dim should be M
    for i, fc in enumerate(self.layers):
      y[..., i, :] = fc(x[..., i, :])
    return y


Instruction: Difference between MultiplicativeLR and LambdaLR

Input:
Can someone explain difference between torch.optim.lr_scheduler.LambdaLR(https://pytorch.org/docs/stable/optim.html) and torch.optim.lr_scheduler.MultiplicativeLR(https://pytorch.org/docs/stable/optim.html)?
Here is brief description of MultiplicativeLR:

and LambdaLR

Output:
The main difference is that they use a different function for computing the learning rate of the function.
LambdaLR's function is:

While MultiplicativeLR's function is:

Thus they would have a different result for the learning rate. Both are useful for a variety of scenarios.


Instruction: How to use "model.trt" in Python

Input:
I have a pytorch model that I exported to ONNX and converted to a tensorflow model with the following command:
trtexec --onnx=model.onnx --batch=400 --saveEngine=model.trt

All of this works, but how do I now load this model.trt in python and run the inference?

Output:
The official documentation has a lot of examples. The basic steps to follow are:

ONNX parser: takes a trained model in ONNX format as input and populates a network object in TensorRT
Builder: takes a network in TensorRT and generates an engine that is optimized for the target platform
Engine: takes input data, performs inferences and emits inference output
Logger: object associated with the builder and engine to capture errors, warnings and other information during the build and inference phases

An example for the engine is:
 import tensorrt as trt
import pycuda.autoinit
import pycuda.driver as cuda
from onnx import ModelProto
import onnx

import numpy as np
import matplotlib.pyplot as plt
from time import time

TRT_LOGGER = trt.Logger(trt.Logger.WARNING)
trt_runtime = trt.Runtime(TRT_LOGGER)

#batch_size = 1
explicit_batch = 1 << (int)(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
#inp_shape = [batch_size, 3, 1024, 1024] # the shape I was using

def build_engine(onnx_path, shape = inp_shape):
    with trt.Builder(TRT_LOGGER) as builder,builder.create_builder_config() as config,\
    builder.create_network(explicit_batch) as network, trt.OnnxParser(network, TRT_LOGGER) as parser:
        if builder.platform_has_fast_fp16:
            builder.fp16_mode = True
        builder.max_workspace_size = (1 << 30)
        #builder.max_workspace_size = (3072 << 20)
        #profile = builder.create_optimization_profile()
        #config.max_workspace_size = (3072 << 20)
        #config.add_optimization_profile(profile)
        print("parsing")
        with open(onnx_path, 'rb') as model:
            print("onnx found")
            if not parser.parse(model.read()):
                print("parse failed")
                for error in range(parser.num_errors):
                    print(parser.get_error(error))
            #parser.parse(model.read())
        last_layer = network.get_layer(network.num_layers - 1)
        # Check if last layer recognizes it's output
        if not last_layer.get_output(0):
            # If not, then mark the output using TensorRT API
            network.mark_output(last_layer.get_output(0))
        network.get_input(0).shape = shape

        engine = builder.build_cuda_engine(network)
        return engine

def save_engine(engine, file_name):
    buf = engine.serialize()
    with open(file_name, 'wb') as f:
        f.write(buf)

def load_engine(trt_runtime, plan_path):
    with open(engine_path, 'rb') as f:
        engine_data = f.read()
    engine = trt_runtime.deserialize_cuda_engine(engine_data)
    return engine

if __name__ == "__main__":
    onnx_path = "./path/to/your/model.onnx"
    engine_name = "./path/to/engine.plan"

    model = ModelProto()
    with open(onnx_path, "rb") as f:
        model.ParseFromString(f.read())

    d0 = model.graph.input[0].type.tensor_type.shape.dim[1].dim_value
    d1 = model.graph.input[0].type.tensor_type.shape.dim[2].dim_value
    d2 = model.graph.input[0].type.tensor_type.shape.dim[3].dim_value
    shape = [batch_size , d0, d1 ,d2]
    print(shape)

    print("trying to build engine")
    engine = build_engine(onnx_path,shape)
    save_engine(engine,engine_name)

    print("finished")

Follow this page for another example and information.


Instruction: Text to Image generation using torch model/path file

Input: I trained a text to image generation model based on https://github.com/aelnouby/Text-to-Image-Synthesis. Now I have 2 path files (one for generator , another for discriminator) . How to generate images using this path files?

Output:
You need to pass your generator path file here.  self.generator.load_state_dict(torch.load(pre_trained_gen))
Refer line 28 of trainer.py