Instruction: Why mini-batches larger than 1 doesn't work, but larger accumulating gradients work?

Input:
I am trying to implement a neural network approximating the logical XOR function, however, the network only converges when using a batch size of 1.
I don't understand why: when I use gradient accumulation with multiple mini-batches of size 1, the convergence is very smooth, but mini-batches of size 2 or more don't work at all.
This issue arises, whatever the learning rate, and I have the same issue with another problem(more complex) than XOR.
I join my code for reference:
import numpy as np
import torch.nn as nn
import torch
import torch.optim as optim
import copy

#very simple network
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc = nn.Linear(2,3,True)
        self.fc1 = nn.Linear(3,1, True)

    def forward(self, x):
        x = torch.sigmoid(self.fc(x))
        x = self.fc1(x)
        return x

def data(n): # return n sets of random XOR inputs and output
    inputs = np.random.randint(0,2,2*n)
    inputs = np.reshape(inputs,(-1,2))
    outputs = np.logical_xor(inputs[:,0], inputs[:,1])
    return torch.tensor(inputs, dtype = torch.float32),torch.tensor(outputs, dtype = torch.float32)

N = 4
net = Net() # first network, is updated with minibatches of size N
net1 = copy.deepcopy(net) # second network, updated with N minibatches of size 1
inputs = torch.tensor([[0,0],[0,1],[1,0],[1,1]], dtype = torch.float32)
labels = torch.tensor([0,1,1,0], dtype = torch.float32)
optimizer = optim.SGD(net.parameters(), lr=0.01)
optimizer1 = optim.SGD(net1.parameters(), lr=0.01)
running_loss = 0
running_loss1 = 0
for epoch in range(25000):  # loop over the dataset multiple times
    # get the inputs; data is a list of [inputs, labels]
    input, labels = data(N)

    # zero the parameter gradients
    optimizer.zero_grad()
    optimizer1.zero_grad()
    # forward + backward + optimize
    loss1_total = 0
    for i in range(N):
        outputs1 = net1(input[i])
        loss1 = (outputs1-labels[i]).pow(2)/N # I divide by N to get the effective mean
        loss1.backward()
        loss1_total += loss1.item()

    outputs = net(input)
    loss = (outputs-labels).pow(2).mean()
    loss.backward()

    # optimization
    optimizer.step()
    optimizer1.step()

    # print statistics
    running_loss += loss.item()
    running_loss1 += loss1_total
    if epoch % 1000 == 999:    # print every 1000 mini-batches
        print(f'[{epoch + 1},  loss: {running_loss/1000 :.3f}, loss1: {running_loss1/1000 :.3f}')
        running_loss1 = 0.0
        running_loss = 0.0

print('Finished Training')
 # exemples of data and outputs for reference ; network 2 always converge to the sub-optimal point(0.5,0.5)
datatest = data(4)
outputs = net(datatest[0])
outputs1 = net1(datatest[0])
inputs = datatest[0]
labels = datatest[1]
print("input",inputs)
print("target",labels)
print("net output",outputs)
print("net output",outputs1)

[EDIT] Improved readability and updated the code
result :
[1000,  loss: 0.259, loss1: 0.258
[2000,  loss: 0.252, loss1: 0.251
[3000,  loss: 0.251, loss1: 0.250
[4000,  loss: 0.252, loss1: 0.250
[5000,  loss: 0.251, loss1: 0.249
[6000,  loss: 0.251, loss1: 0.247
[7000,  loss: 0.252, loss1: 0.246
[8000,  loss: 0.251, loss1: 0.244
[9000,  loss: 0.252, loss1: 0.241
[10000,  loss: 0.251, loss1: 0.236
[11000,  loss: 0.252, loss1: 0.230
[12000,  loss: 0.252, loss1: 0.221
[13000,  loss: 0.250, loss1: 0.208
[14000,  loss: 0.251, loss1: 0.193
[15000,  loss: 0.251, loss1: 0.175
[16000,  loss: 0.251, loss1: 0.152
[17000,  loss: 0.252, loss1: 0.127
[18000,  loss: 0.251, loss1: 0.099
[19000,  loss: 0.251, loss1: 0.071
[20000,  loss: 0.251, loss1: 0.048
[21000,  loss: 0.251, loss1: 0.029
[22000,  loss: 0.251, loss1: 0.016
[23000,  loss: 0.250, loss1: 0.008
[24000,  loss: 0.251, loss1: 0.004
[25000,  loss: 0.251, loss1: 0.002

Finished Training

input tensor([[1., 0.],
        [0., 0.],
        [0., 0.],
        [0., 0.]])
target tensor([1., 0., 0., 0.])
net output tensor([[0.4686],
        [0.4472],
        [0.4472],
        [0.4472]], grad_fn=<AddmmBackward0>)
net1 output tensor([[0.9665],
        [0.0193],
        [0.0193],
        [0.0193]], grad_fn=<AddmmBackward0>)

Please, could you explain me why this strange phenomena is appearing ? I searched for a long time on the net, without success...
Excuse me if my question is not well formatted, it is the first time I ask a question on stack overflow.
EDIT :
I found, comparing accumulated gradients of size 1 minibatches and gradients from minibatches of size N, that the computed gradients are mostly the same, only small(but noticeable) differences appear probably due to approximation errors, so my implementation looks fine at first sight. I still don't get where does this strong convergence property of minibatches of size 1 come from.

Output:
The problem lies in the way you define labels / compute the loss in
 loss = (outputs-labels).pow(2).mean()

We have labels.shape = [4] but outputs.shape =[4, 1]. This due to the broadcasting, the difference
(outputs - labels).shape = [4, 4]

which means we compute all pairwise differences between outputs and labels (and then take their 2nd power and average them), which basically means that no meaningful supervision takes place.
The quick way to fix that here would be adding a dummy dimension here:
loss = (outputs-labels[:, None]).pow(2).mean()

but the clean way would be doing it the correct way right from the start, that is defining your labels in a ways such that labels.shape = [_, 1]:
labels = torch.tensor([[0], [1], [1], [0]], dtype=torch.float32)

(and similar in your data() function).


Instruction: How can I make my CNN output_feature = 1 since I am testing for 0 or 1

Input:
class BreastCancerCNN(ImageClassificationBase):
    def __init__(self):
        super().__init__()
        self.network = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 64 x 16 x 16

            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 128 x 8 x 8

            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 256 x 4 x 4

            nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(),
            nn.MaxPool2d(2, 2), # output: 512 x 2 x 2

            nn.Flatten(),
            nn.Linear(256*2*2, 512),
            nn.ReLU(),
            nn.Linear(512, 256),
            nn.ReLU(),
            nn.Linear(256, 1)

        )

    def forward(self, xb):
        return self.network(xb)

@torch.no_grad()

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']

def evaluate(model, val_loader):
    model.eval()
    outputs = [model.validation_step(batch) for batch in val_loader]
    print(outputs)
    return model.validation_epoch_end(outputs)

def fit(epochs, lr, model, train_loader, val_loader, opt_func=torch.optim.SGD):
    history = []

    # Set up cutom optimizer with weight decay
    optimizer = opt_func(model.parameters(), lr, weight_decay=0)
    # Set up one-cycle learning rate scheduler
    sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs,
                                                steps_per_epoch=len(train_loader))
    for epoch in range(epochs):
        train_losses = []
        lrs = []
        # Training Phase
        model.train()
        train_losses = []
        for batch in train_loader:
            loss = model.training_step(batch)
            train_losses.append(loss)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()

            # Record & update learning rate
            lrs.append(get_lr(optimizer))
            sched.step()

        # Validation phase
        result = evaluate(model, val_loader)
        result['train_loss'] = torch.stack(train_losses).mean().item()
        result['lrs'] = lrs
        model.epoch_end(epoch, result)
#         print(result)
        history.append(result)
    return history

Get data from the history after running one epoch
history = [evaluate(model, val_dl)]
history
This is the code that results in
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_33/1481765314.py in <module>
      1 # Get data from the histrory after running one epoch
----> 2 history = [evaluate(model, val_dl)]
      3 history

/tmp/ipykernel_33/559867535.py in evaluate(model, val_loader)
      7 def evaluate(model, val_loader):
      8     model.eval()
----> 9     outputs = [model.validation_step(batch) for batch in val_loader]
     10     print(outputs)
     11     return model.validation_epoch_end(outputs)

/tmp/ipykernel_33/559867535.py in <listcomp>(.0)
      7 def evaluate(model, val_loader):
      8     model.eval()
----> 9     outputs = [model.validation_step(batch) for batch in val_loader]
     10     print(outputs)
     11     return model.validation_epoch_end(outputs)

/tmp/ipykernel_33/1785597948.py in validation_step(self, batch)
      8     def validation_step(self, batch):
      9         images, labels = batch
---> 10         out = self(images)                    # Generate predictions
     11         loss = F.cross_entropy(out, labels)   # Calculate loss
     12         acc = accuracy(out, labels)           # Calculate accuracy

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/tmp/ipykernel_33/3839242004.py in forward(self, xb)
     45
     46     def forward(self, xb):
---> 47         return self.network(xb)

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    139     def forward(self, input):
    140         for module in self:
--> 141             input = module(input)
    142         return input
    143

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/linear.py in forward(self, input)
    101
    102     def forward(self, input: Tensor) -> Tensor:
--> 103         return F.linear(input, self.weight, self.bias)
    104
    105     def extra_repr(self) -> str:

RuntimeError: mat1 and mat2 shapes cannot be multiplied (100x2048 and 1024x512)

I would like to analyse images for a defect and thus have a scale of 0 to
1 so as to evaluate the presence of a defect in the sample images
Please help me break it down and understand what is actually going on and how do I determine the right values for the linear to get a single output that will result in a prediction of a 1 or 0

Output:
Without going fully through your code, but looking at the comments and error messages, one can see that for the last maxpool layer:
nn.MaxPool2d(2, 2), # output: 512 x 2 x 2

That is, the number of features per image are 512 x 2 x 2 = 1024.
This is then flattened and passed through this linear layer:
nn.Linear(256*2*2, 512),

But this only accepts 256 x 2 x 2 = 1024 input features. This is what causes the mismatch when calling forward for this linear layer. To fix this, change this statement to:
nn.Linear(512*2*2, 512),


Instruction: Conda install conflicting requirements

Input:
I am attempting to use Conda to create an environment from a Pip requirements file. The contents of the file are
requirements.txt
numpy==1.18.2
torch==1.4.0
torchvision==0.5.0
scikit-learn==0.22.2.post1
Pillow==8.3.2
pydicom==1.4.2
pandas==1.0.3

Running the command
conda create -n $name --file requirements.txt

gives a PackageNotFound error as the channels are missing.
How do I amend this?

Output:
Possible Issues
There are a few potential issues.
Conda pytorch
First, not all packages in Conda go by the same name as they do in other repositories. Part of this is due to the nature of Conda being a general package repository, rather than a language-specific one. In particular, the torch module is delivered via the Conda pytorch package.
So that has to change.
NumPy version unavailable
That particular build of NumPy does not appear to be available in either defaults or conda-forge channels.
$ mamba search numpy=1.18.2
No match found for: numpy=1.18.2. Search: *numpy*=1.18.2

PackagesNotFoundError: The following packages are not available from current channels:

  - numpy=1.18.2

Current channels:

  - https://conda.anaconda.org/conda-forge/osx-64
  - https://conda.anaconda.org/conda-forge/noarch
  - https://conda.anaconda.org/bioconda/osx-64
  - https://conda.anaconda.org/bioconda/noarch
  - https://repo.anaconda.com/pkgs/main/osx-64
  - https://repo.anaconda.com/pkgs/main/noarch
  - https://repo.anaconda.com/pkgs/r/osx-64
  - https://repo.anaconda.com/pkgs/r/noarch

Why would this happen? For most Python packages, Conda works downstream of the PyPI repository. When new releases come out, the Conda Forge bot (for example) will auto-generate a pull request to corresponding feedstock. Sometimes these don't "just work" and need some troubleshooting to get built. Occasionally, the process to get the builds working won't finish before a new release hits. This results in a newer pull request superceding the previous one, and can lead to the old pull request being abandoned. This results in gaps in the coverage of PyPI by Conda Forge, and is exactly what happened here.
If you can tolerate a different version, conda-forge does provide v1.18.1 (below) and v1.18.4 (above).
Otherwise, if you require exact replication of package versions, then you will have to source this from PyPI. I'll show this in the end.
Channel issues
Missing channels
OP does not indicate the channel configuration. The torchvision==0.5.0 package, for example, only is available through the pytorch channel.
Masked channels
Another issue here could be the use of the channel_priority: strict setting. If this setting were used, it is possible a channel with the version required might be a priori excluded by the SAT solver simply because the package (but not the correct version) is available in a higher priority channel. These days channel_priority: flexible is the default and can be set with:
conda config --set channel_priority flexible

Solutions
Exact replication (PyPI only)
Give the package names and versions, these packages likely originated from PyPI. If you need to exactly replicate the original environment - say, for reproducing scientific results - then I'd recommend sourcing everything from PyPI. The best way to do this is to use Conda to source Python and Pip, then let Pip install the requirements.txt.
Judging from the package versions, we're talking Python 3.7 or 3.8. You'd probably be fine with just python=3.8, but [a precise guesstimate from release dates would be python=3.8.2. So, try something like:
environment.yaml
name: my_env
channels:
  - conda-forge
dependencies:
  - python=3.8.2
  - pip
  - pip:
    - -r requirements.txt

Then create the environment with
conda env create -n $name -f environment.yaml

making sure the requirements.txt is in the folder with the YAML.
If adding packages to this environment later, I would recommend only using pip install. Otherwise, Conda may have issues.
Conda-only environment
Assuming the numpy=1.18.2 can be substituted, a Conda-only environment might be something like:
environment.yaml
name: my_env
channels:
  - pytorch
  - conda-forge
dependencies:
  - python=3.8
  - numpy=1.18.1  # alternatively, 1.18.4
  - pytorch=1.4.0
  - torchvision=0.5.0
  - scikit-learn=0.22.2.post1
  - pillow=8.3.2
  - pydicom=1.4.2
  - pandas=1.0.3

Again, creating with:
conda env create -n $name -f environment.yaml

Note that in YAML only one = is used. This would be the best approach if you plan to install additional packages through Conda in an ad hoc manner (e.g., conda install).
Mixed Conda-Pip environment
You could also try a mixed environment mostly similar to the last one, but having Pip specifically provide numpy==1.18.2. I wouldn't recommend this, since the other dependencies with definitely bring in NumPy first from Conda, and then Pip will clobber it to provide the exact version.


Instruction: Runtime Error: mat1 and mat2 shapes cannot be multiplied (62x2304 and 1568x3)

Input:
I am un able to find error input 32*32 gray images:
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(
                in_channels=1, # gray-scale images
                out_channels=16,
                kernel_size=5, # 5x5 convolutional kernel
                stride=1,  #no. of pixels pass at a time
                padding=2, # to preserve size of input image
            ),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2),
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(16, 32, 5, 1, 2),
            nn.ReLU(),
            nn.MaxPool2d(2),
        )
        # fully connected layers
        self.out = nn.Linear(32*7*7, 3)
    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        # flatten the output of conv2
        x = x.view(x.size(0), -1)
        output = self.out(x)
        return output
cnn=CNN()
cnn

Output:
Your linear layer expects input of size 32x7x7. Given that your conv1 and conv2 layers performs max pooling with stride=2, that means your network is configured for input size of 28x28 (MNIST usual input size) and not 32x32 as you expect.
Moreover, considering the values in your error message (64x2304) I assume you are working with batch_size=64, but your images are NOT 32x32, but rather 32x?? which is slightly larger than 32, resulting with a feature map of 32x8x9 after the pooling.


Instruction: Different results in computing KL Divergence using Pytorch Distributions vs manually

Input:
I noticed the KL-Divergence term KL(Q(x)||P(x)) is computed differently when using
mean(Q(x)*(log Q(x) - log P(x)))

vs
torch.distributions.kl_divergence(Q, P)

where
Q = torch.distributions.Normal(some mean, some sigma)
P = torch.distributions.Normal(0, 1)

and when I plot the KL-divergence losses, I get this two similar but different plots:
here
Can anyone point out what is causing this difference?
The full code is below:
import numpy as np
import torch
import torch.distributions as dist
import matplotlib.pyplot as plt

def kl_1(log_qx, log_px):
    """
    inputs: [B, z_dim] torch
    """
    return (log_qx.exp() * (log_qx-log_px)).mean()

# ground-truth (target) P(x)
P = dist.Normal(0, 1)

mus = np.arange(-5, 5, 0.1)
sigma = 1
N = 100
kls = {"1": [], "2": []}
for mu in mus:
    # prediction (current) Q(x)
    Q = dist.Normal(mu, sigma)

    # sample from Q
    qx = Q.sample((N,))

    # log prob
    log_qx = Q.log_prob(qx)
    log_px = P.log_prob(qx)

    # kl 1
    kl1 = kl_1(log_qx, log_px)
    kls['1'].append(kl1.numpy())

    # kl 2
    kl2 = dist.kl_divergence(Q, P)
    kls['2'].append(kl2.numpy())

plt.figure()
plt.scatter(mus, kls['1'], label="Q*(logQ-logP)")
plt.scatter(mus, kls['2'], label="kl_divergence")
plt.xlabel("mean of Q(x)")
plt.ylabel("computed KL Divergence")
plt.legend()
plt.show()

Output:
You have the sample weighted by the probability density if you are computing the expected value from an integral on dx. If you are using a sample from the given distribution then you approximate the expected value as the mean directly, that corresponds to integration on d cq(x) thus d cq(x) = q(x) dx, where cq(x) is the cumulative probability function, and q(x)  id the probability density funciton of the variable Q.
import numpy as np
import torch
import torch.distributions as dist
import matplotlib.pyplot as plt

def kl_1(log_qx, log_px):
    """
    inputs: [B, z_dim] torch
    """
    return (log_qx-log_px).mean()

# ground-truth (target) P(x)
P = dist.Normal(0, 1)

mus = np.arange(-5, 5, 0.1)
sigma = 1
N = 100
kls = {"1": [], "2": []}
for mu in mus:
    # prediction (current) Q(x)
    Q = dist.Normal(mu, sigma)

    # sample from Q
    qx = Q.sample((N,))

    # log prob
    log_qx = Q.log_prob(qx)
    log_px = P.log_prob(qx)

    # kl 1
    kl1 = kl_1(log_qx, log_px)
    kls['1'].append(kl1.numpy())

    # kl 2
    kl2 = dist.kl_divergence(Q, P)
    kls['2'].append(kl2.numpy())

plt.figure()
plt.scatter(mus, kls['1'], label="Q*(logQ-logP)")
plt.scatter(mus, kls['2'], label="kl_divergence")
plt.xlabel("mean of Q(x)")
plt.ylabel("computed KL Divergence")
plt.legend()


Instruction: How to apply the parameters (weights of a network) to a vector of inputs using PyTorch?

Input:
I have a simple "Neural Network" that takes two inputs and computes a linear combination and outputs a scalar. In the case of one single equation, i.e., in 1D, we would have: weight1*inputA + weight2*inputB = output. Now, I am interested in multi-dimensional inputs, i.e., that inputA is a vector instead. So I want my network to apply the weight1 to a vector, for example: weight1 * [input1, input2] + weight2 * [input3, input4]. In this setting, I would want the output to be a vector too: [out1, out2], where out1 = weight1*inputA + weight2*inputB. However, I don't want to change the input and output dimensions of my network. As in the 1D case, I would initialise the network as net = LinearNet(2,1), since we take two inputs, inputA and inputB, and get one output. I understand that the input and output itself are multi-dimensional, but this should not bother my network.
Below is a min. working example:
import numpy as np
import torch
from torch import nn

def f(x,a,b,c,d):
    x_next = np.zeros((2,))
    x_next[0] = x[0]*(a-b*x[1])
    x_next[1] = -x[1]*(c-d*x[0])
    return x_next #returns [2,:]

a = 1
b = 0.5
c = 1
d = 0.5
x01 = 1 #init cond. varA
x02 = 2 #init cond. varB
params = [a,b,c,d,x01,x02]

# ==================
h = 0.001
T = 10
K = int(T/h)

# forward euler approx.
x_traj_FE = np.zeros((2,K))
a, b, c, d, x_traj_FE[0,0], x_traj_FE[1,0] = params
for k in range(K-1):
    x_traj_FE[:,k+1] = x_traj_FE[:,k] + h*f(x_traj_FE[:,k],a,b,c,d)

# ==================
class LinearNet(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LinearNet, self).__init__()

        self.layer1 = nn.Linear(input_dim, output_dim, bias=False)

        #no activation function

    def forward(self, x):
        x = self.layer1(x)
        #no activation
        return x

torch.manual_seed(789)
net = LinearNet(2,1)

x_traj = np.zeros((2,K))
a, b, c, d, x_traj[0,0], x_traj[1,0] = params
for k in range(K-1):
    print(net.layer1.weight)
    input1 = x_traj[:,k]
    input2 = f(x_traj[:,k],a,b,c,d)
    inputs = torch.Tensor(np.array([input1, input2]))
    print('input:  '+str(inputs))
    print('shape:  '+str(inputs.shape))
    print('output: '+str(net(inputs)))
    break

If we run this, we will receive the following output:
Parameter containing:
tensor([[0.3871, 0.5595]], requires_grad=True)
input:  tensor([[ 1.,  2.],
        [ 0., -1.]])
shape:  torch.Size([2, 2])
output: tensor([[ 1.5060],
        [-0.5595]], grad_fn=<MmBackward0>)

Eventually, I would like to compute one forward euler step of the coupled ode f, where the input1 is the initial conditions of each equation within the system f, and input2 is the system evaluated at the previous step. If we do it manually, one step can be calculated as (given the weights from the network above):
weights = [0.3871, 0.5595]
x_traj = np.zeros((2,K))
a, b, c, d, x_traj[0,0], x_traj[1,0] = params
for k in range(K-1):
    x_traj[:,k+1] = weights[0]*x_traj[:,k] + weights[1]*f(x_traj[:,k],a,b,c,d)
    break
x_traj

And the output is:
array([[1.    , 0.3871, 0.    , ..., 0.    , 0.    , 0.    ],
       [2.    , 0.2147, 0.    , ..., 0.    , 0.    , 0.    ]])

As we see, the output of the network differs from the manual computation. I don't see how the network computes this scalar-vector multiplication and hence, I can't understand how to retrieve the same output as with the manual computation.

Output:
When you do inputs = torch.Tensor(np.array([input1, input2])), you are creating a tensor where the first row is input1 and the second row is input2. When you are multiplying manually, your first column is equal to what was input1 previously, and similarly for input2. In order to get equivalent results to the first case, you could change the statement in the loop to (I do not know which of the two is logically correct in your use case, however):
inp = torch.stack([x_traj[:,k], f(x_traj[:,k],a,b,c,d)],dim=0) # This gives the same matrix as `inputs` in the first case
x_traj[:,k+1] = weights[0]*inp[:,0] + weights[1]*inp[:,1] # This is equivalent to calling the network in the first case


Instruction: What does self.model(batch) do in pytorch?

Input:
I am currently rewriting a PyTorch code to tensorflow. During this I found a line that I don't understand, so I am not able to translate it to tensorflow.
Can someone explain me what this does/ means?
self.model(batch)

Output:
It seems like self.model is a layer/layers of a neural network, derived from nn.Module class.
The call self.model(batch) invoke's self.model's __call__ method with the argument batch.
If you inspect closely, nn.Module.__call__ do some "bookkeeping" but essentially, it calls the self.model's forward function.


Instruction: How to plot loss when using HugginFace's Trainer?

Input:
While finetuning a model using HF's trainer.
training_args = TrainingArguments(output_dir=data_dir + "test_trainer")

metric = load_metric("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

training_args = TrainingArguments(num_train_epochs=5,per_device_train_batch_size=64,per_device_eval_batch_size=32,output_dir="test_trainer", evaluation_strategy="epoch")

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
    compute_metrics=compute_metrics,
)
trainer.train()

How would I be able to plot the loss in a notebook?
(Perhaps Is it possible to get a list of the loss)

Output: It is possible to get a list of losses. You can access the history of logs after training is complete with: trainer.state.log_history


Instruction: Why CUDA is unavailable for using with easyocr?

Input:
According to Pytorch I used this command in cmd
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113

But CUDA still unavailable. Could someone help me, please?
import torch
print(torch.cuda.is_available())

The output will be False

Output:
You have to update driver first:
Here is a concept diagram from nvidia website

Here is another one:

More at CUDA Compatibility


Instruction: Meaning of stacklevel in PyTorch

Input:
I came across many function in PyTorch that have _stacklevel as argument. Here an example of the Softmax module's forward() method where it is used:
def forward(self, input: Tensor) -> Tensor:
    return F.softmax(input, self.dim, _stacklevel=5)

What does _stacklevel mean? What is it good for?

Output:
stacklevel is used in python to indicate warning mechanism how far up the stack it has to go to find the line that called the function which issued the warning. For example, the code below makes the warning refer to deprecation()’s caller by using stacklevel=2, rather than to the source of deprecation() itself. stacklevel=3 would refer to the caller of deprecation()’s caller and so on.
def deprecation(message):
    warnings.warn(message, DeprecationWarning, stacklevel=2)

See this page for more information.
Regarding the specific case you mention, in PyTorch's F.softmax, F.softmin, and F.log_softmax functions, this argument is related to the warning issued when dim is not specified. However, it seems that it should be dropped since legacy softmax dim behavior is gone, or at least clarified in the documentation. At the moment, this is only mentioned on the following open issues from pytorch repo:

pytorch/issues/36524
pytorch/issues/64038

It will probably be fixed or clarified in the future, but for the moment my recommendation is to simply ignore it.


Instruction: Understanding gradient computation using backward() in PyTorch

Input:
I'm trying to understand the basic pytorch autograd system:
x = torch.tensor(10., requires_grad=True)
print('tensor:',x)
x.backward()
print('gradient:',x.grad)

output:
tensor: tensor(10., requires_grad=True)
gradient: tensor(1.)

since x is a scalar constant and no function is applied to it, I expected 0. as the gradient output. Why is the gradient 1. instead?

Output:
Whenever you are using value.backward(), you compute the derivative value (in your case value == x) with respect to all your parameters (in your case that is just x). Roughly speaking, this means all tensors that are somehow involved in your computation that have requires_grad=True. So this means
x.grad = dx / dx = 1

To add to that: With the automatic differentiation you always ever compute with "constant" values: All your function or networks are always evaluated at a concrete point. And the gradient you get is the gradient evaluated at that same point. There is no symbolic computation taking place. All the information needed for the computation of the gradient is encoded in the computation graph.


Instruction: pytorch dataloader - RuntimeError: stack expects each tensor to be equal size, but got [157] at entry 0 and [154] at entry 1

Input:
I am a beginner with pytorch. I am trying to do an aspect based sentiment analysis. I am facing the error mentioned in the subject. My code is as follows: I request help to resolve this error. Thanks in advance. I will share the entire code and the error stack.
!pip install transformers
import transformers
from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup
import torch
import numpy as np
import pandas as pd
import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib import rc
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from collections import defaultdict
from textwrap import wrap
from torch import nn, optim
from torch.utils.data import Dataset, DataLoader
%matplotlib inline
%config InlineBackend.figure_format='retina'
sns.set(style='whitegrid', palette='muted', font_scale=1.2)
HAPPY_COLORS_PALETTE = ["#01BEFE", "#FFDD00", "#FF7D00", "#FF006D", "#ADFF02", "#8F00FF"]
sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))
rcParams['figure.figsize'] = 12, 8
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
torch.manual_seed(RANDOM_SEED)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

df = pd.read_csv("/Users/user1/Downloads/auto_bio_copy.csv")
I am importing a csv file which has content and label as shown below:
df.head()
                     content                                      label
0   I told him I would leave the car and come back...   O O O O O O O O O O O O O O O O O O O O O O O ...
1   I had the ignition interlock device installed ...   O O O B-Negative I-Negative I-Negative O O O O...
2   Aug. 23 or 24 I went to Walmart auto service d...   O O O O O O O B-Negative I-Negative I-Negative...
3   Side note This is the same reaction I 'd gotte...   O O O O O O O O O O O O O O O O O O O O O O O ...
4   Locked out of my car . Called for help 215pm w...   O O O O O O O O O O O O O O O O O B-Negative O...

df.shape
(1999, 2)
I am converting the label values into integers as follows:
O=zero(0), B-Positive=1, I-Positive=2, B-Negative=3, I-Negative=4, B-Neutral=5, I-Neutral=6, B-Mixed=7, I-Mixed=8
df['label'] = df.label.str.replace('O', '0')
df['label'] = df.label.str.replace('B-Positive', '1')
df['label'] = df.label.str.replace('I-Positive', '2')
df['label'] = df.label.str.replace('B-Negative', '3')
df['label'] = df.label.str.replace('I-Negative', '4')
df['label'] = df.label.str.replace('B-Neutral', '5')
df['label'] = df.label.str.replace('I-Neutral', '6')
df['label'] = df.label.str.replace('B-Mixed', '7')
df['label'] = df.label.str.replace('I-Mixed', '8')

Next, converting the string to integer list as follows:
df['label'] = df['label'].str.split(' ').apply(lambda s: list(map(int, s)))

df.head()

                     content                                         label
0   I told him I would leave the car and come back...   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
1   I had the ignition interlock device installed ...   [0, 0, 0, 3, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
2   Aug. 23 or 24 I went to Walmart auto service d...   [0, 0, 0, 0, 0, 0, 0, 3, 4, 4, 4, 0, 0, 0, 0, ...
3   Side note This is the same reaction I 'd gotte...   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
4   Locked out of my car . Called for help 215pm w...   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...

PRE_TRAINED_MODEL_NAME = 'bert-base-cased'

tokenizer = BertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)

token_lens = []
for txt in df.content:
  tokens = tokenizer.encode_plus(txt, max_length=512, add_special_tokens=True, truncation=True, return_attention_mask=True)
  token_lens.append(len(tokens))
MAX_LEN = 512

class Auto_Bio_Dataset(Dataset):
    def __init__(self, contents, labels, tokenizer, max_len):
        self.contents = contents
        self.labels = labels
        self.tokenizer = tokenizer
        self.max_len = max_len
    def __len__(self):
        return len(self.contents)
    def __getitem__(self, item):
        content = str(self.contents[item])
        label = self.labels[item]
        encoding = self.tokenizer.encode_plus(
          content,
          add_special_tokens=True,
          max_length=self.max_len,
          return_token_type_ids=False,
          #padding='max_length',
          pad_to_max_length=True,
          truncation=True,
          return_attention_mask=True,
          return_tensors='pt'
        )
        return {
          'content_text': content,
          'input_ids': encoding['input_ids'].flatten(),
          'attention_mask': encoding['attention_mask'].flatten(),
          'labels': torch.tensor(label)
        }

df_train, df_test = train_test_split(
  df,
  test_size=0.1,
  random_state=RANDOM_SEED
)
df_val, df_test = train_test_split(
  df_test,
  test_size=0.5,
  random_state=RANDOM_SEED
)

df_train.shape, df_val.shape, df_test.shape

((1799, 2), (100, 2), (100, 2))

def create_data_loader(df, tokenizer, max_len, batch_size):
    ds = Auto_Bio_Dataset(
        contents=df.content.to_numpy(),
        labels=df.label.to_numpy(),
        tokenizer=tokenizer,
        max_len=max_len
  )
    return DataLoader(
        ds,
        batch_size=batch_size,
        num_workers=2
  )

BATCH_SIZE = 16
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)

data = next(iter(train_data_loader))
data.keys()

Error is as follows:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-71-e0a71018e473> in <module>
----> 1 data = next(iter(train_data_loader))
      2 data.keys()

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in __next__(self)
    528             if self._sampler_iter is None:
    529                 self._reset()
--> 530             data = self._next_data()
    531             self._num_yielded += 1
    532             if self._dataset_kind == _DatasetKind.Iterable and \

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _next_data(self)
   1222             else:
   1223                 del self._task_info[idx]
-> 1224                 return self._process_data(data)
   1225
   1226     def _try_put_index(self):

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _process_data(self, data)
   1248         self._try_put_index()
   1249         if isinstance(data, ExceptionWrapper):
-> 1250             data.reraise()
   1251         return data
   1252

~/opt/anaconda3/lib/python3.7/site-packages/torch/_utils.py in reraise(self)
    455             # instantiate since we don't know how to
    456             raise RuntimeError(msg) from None
--> 457         raise exception
    458
    459

RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch
    return self.collate_fn(data)
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 157, in default_collate
    return elem_type({key: default_collate([d[key] for d in batch]) for key in elem})
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 157, in <dictcomp>
    return elem_type({key: default_collate([d[key] for d in batch]) for key in elem})
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 138, in default_collate
    return torch.stack(batch, 0, out=out)
RuntimeError: stack expects each tensor to be equal size, but got [157] at entry 0 and [154] at entry 1

I found in some github post that this error can be because of batch size, so i changed the batch size to 8 and then the error is as follows:
BATCH_SIZE = 8
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_val, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)

data = next(iter(train_data_loader))
data.keys()

RuntimeError                              Traceback (most recent call last)
<ipython-input-73-e0a71018e473> in <module>
----> 1 data = next(iter(train_data_loader))
      2 data.keys()

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in __next__(self)
    528             if self._sampler_iter is None:
    529                 self._reset()
--> 530             data = self._next_data()
    531             self._num_yielded += 1
    532             if self._dataset_kind == _DatasetKind.Iterable and \

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _next_data(self)
   1222             else:
   1223                 del self._task_info[idx]
-> 1224                 return self._process_data(data)
   1225
   1226     def _try_put_index(self):

~/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _process_data(self, data)
   1248         self._try_put_index()
   1249         if isinstance(data, ExceptionWrapper):
-> 1250             data.reraise()
   1251         return data
   1252

~/opt/anaconda3/lib/python3.7/site-packages/torch/_utils.py in reraise(self)
    455             # instantiate since we don't know how to
    456             raise RuntimeError(msg) from None
--> 457         raise exception
    458
    459

RuntimeError: Caught RuntimeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
    data = fetcher.fetch(index)
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch
    return self.collate_fn(data)
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 157, in default_collate
    return elem_type({key: default_collate([d[key] for d in batch]) for key in elem})
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 157, in <dictcomp>
    return elem_type({key: default_collate([d[key] for d in batch]) for key in elem})
  File "/Users/namrathabhandarkar/opt/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 137, in default_collate
    out = elem.new(storage).resize_(len(batch), *list(elem.size()))
RuntimeError: Trying to resize storage that is not resizable

I am not sure what is causing the first error(the one mentioned in subject). I am using padding and truncate in my code, yet the error.
Any help to resolve this issue is highly appreciated.
Thanks in advance.

Output:
Quick answer: you need to implement your own collate_fn function when creating a DataLoader. See the discussion from PyTorch forum.
You should be able to pass the function object to DataLoader instantiation:
def my_collate_fn(data):
    # TODO: Implement your function
    # But I guess in your case it should be:
    return tuple(data)

return DataLoader(
    ds,
    batch_size=batch_size,
    num_workers=2,
    collate_fn=my_collate_fn
)

This should be the way to solving this, but as a temporary remedy in case anything is urgent or a quick test is nice, simply change batch_size to 1 to prevent torch from trying to stack things with different shapes up.


Instruction: manually download pytorch and feed downloaded path to anaconda

Input:
I tried to conda install pytorch ... but it doesn't even start and gives me
("Connection broken: ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)", ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))

I'm thinking to download the necessary file manually(safe download even if internet is slow) and then set it to anaconda so that installs directly from downloaded file instead of downloading it self

Output:
This can be done by downloading pip wheels for torch, torchvision and torchaudio etc. and doing pip installation for local wheels. Suitable Pytorch wheels can be downloaded from here. Now, you can simply pip install the downloaded whl files:
pip install file.whl

Here file.whl is the downloaded wheel.


Instruction: pytorch inplace runtime error with class inheritance

Input:
May I know why this forward() function gives runtime error on inplace operation ?
Note: I have done some code debugging which leads to the following line of code:
class ConvEdge(Edge):
    def __init__(self, stride):
        super().__init__()
        self.f = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=(3, 3), stride=(stride, stride), padding=1)

If you guys have a look at the code snippet regarding class ConvEdge(Edge) , I am actually having second thought on how inheritance is being viewed and processed by pytorch autograd library.
What do you guys think ?

Output: the issue is resolved using with torch.no_grad() which basically does not propagate the gradient when it is not necessary.


Instruction: Multilabel classification using ResNet - loading sample using the dataloader takes a long time

Input:
I need to implement a ResNet-based multiclass classifier, and I am using this notebook as a starting point. At the moment, I am just going through the notebook, checking that all steps work fine. When I try to load a sample with the dataloader (command: sample = next(iter(train_loader))), I get no results even after waiting for more than an hour. Why is that?
The dataloader is defined in this cell:
#Pre-processing transformations
data_transforms = transforms.Compose([
        transforms.Resize((224,224)),
        transforms.ToTensor(),
        transforms.Normalize((0.5,0.5,0.5), (0.5,0.5,0.5))
    ])

#Getting the data
cardata = CarDataset("./content/content/carimages/car_ims", transform=data_transforms,translation_dict=translation_dict)

#Split the data in training and testing
train_len = int(cardata.__len__()*0.8)
test_len = int(cardata.__len__()*0.2)
train_set, val_set = torch.utils.data.random_split(cardata, [train_len, test_len])

#Create the dataloader for each dataset
train_loader = DataLoader(train_set, batch_size=16, shuffle=True,
                                num_workers=4, drop_last=True)
test_loader = DataLoader(val_set, batch_size=16, shuffle=False,
                               num_workers=4, drop_last=True)

If I try loading the data on the GPU using
train_set.cardata.to(torch.device("cuda:0"))  # put data into GPU entirely
train_set.to(torch.device("cuda:0"))

I get the error 'Subset' object has no attribute 'cardata'. Am I doing something wrong? Is it just normal that the dataloader takes so long to load the dataset images? Thanks!

Output:
It seems there is no error at calling train/val set and loader of provided code. There might be an error inside of CarDataset() class, or maybe setting num_workers=0 could help.
Also, please avoid acquiring a sample via next(iter(train_loader)). This creates a new loader every time it is called. Use train_iter = iter(train_loader) and inputs, labels = train_iter.next() instead.
Additionally, as far as I know, putting the entire dataset to a GPU or GPUs is not supported. You can load a mini-batch data (pytorch tensor) to it: inputs = inputs.to(torch.device("cuda"))
Any advise or correcting a wrong answer is welcomed.


Instruction: Change all images in training set

Input:
I have a convolutional neural network. And I wanted to train it on images from the training set but first they should be wrapped with my function change(tensor, float) that takes in a tensor/image of the form [hight,width,3] and a float.
Batch size =4
loading data
trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
                                          shuffle=True, num_workers=2)

Cnn architecture
for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data
        #size of inputs [4,3,32,32]
        #size of labels [4]

        inputs = change(inputs,0.1) <----------------------------

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs) #[4, 10]

        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')
            running_loss = 0.0

print('Finished Training')

I am trying to apply the image function change but it gives an object error.
it there a quick way to fix it?
I am using a Julia function but it works completely fine with other objects. Error message:
JULIA: MethodError: no method matching copy(::PyObject)
Closest candidates are:
  copy(!Matched::T) where T<:SHA.SHA3_CTX at /opt/julia-1.7.2/share/julia/stdlib/v1.7/SHA/src/types.jl:213
  copy(!Matched::T) where T<:SHA.SHA2_CTX at /opt/julia-1.7.2/share/julia/stdlib/v1.7/SHA/src/types.jl:212
  copy(!Matched::Number) at /opt/julia-1.7.2/share/julia/base/number.jl:113

Output:
I would recommend to put change function to transforms list, so you do data changes on transformation stage.
partial from functools will help you to fix number of arguments, like this:
from functools import partial

def change(input, float):
    pass

# Use partial to fix number of params, such that change accepts only input
change_partial = partial(change, float=pass_float_value_here)

# Add change_partial to a list of transforms before or after converting to tensors
transforms = Compose([
    RandomResizedCrop(img_size),  # example
    # Add change_partial here if it operates on PIL Image
    change_partial,
    ToTensor(),  # convert to tensor
    # Add change_partial here if it operates on torch tensors
    change_partial,
])


Instruction: Reshape tensors in pytorch?

Input:
I'm struggling with the result of a matrix multiplication in pytorch and I don't know how to solve it, in particular:
I'm multiplying these two matrices
tensor([[[[209.5000, 222.7500],
          [276.5000, 289.7500]],

         [[208.5000, 221.7500],
          [275.5000, 288.7500]]]], dtype=torch.float64)

and
tensor([[[[ 0.,  1.,  2.,  5.,  6.,  7., 10., 11., 12.],
          [ 2.,  3.,  4.,  7.,  8.,  9., 12., 13., 14.],
          [10., 11., 12., 15., 16., 17., 20., 21., 22.],
          [12., 13., 14., 17., 18., 19., 22., 23., 24.]],

         [[25., 26., 27., 30., 31., 32., 35., 36., 37.],
          [27., 28., 29., 32., 33., 34., 37., 38., 39.],
          [35., 36., 37., 40., 41., 42., 45., 46., 47.],
          [37., 38., 39., 42., 43., 44., 47., 48., 49.]],

         [[50., 51., 52., 55., 56., 57., 60., 61., 62.],
          [52., 53., 54., 57., 58., 59., 62., 63., 64.],
          [60., 61., 62., 65., 66., 67., 70., 71., 72.],
          [62., 63., 64., 67., 68., 69., 72., 73., 74.]]]],
       dtype=torch.float64)

with the following line of code  A.view(2,-1) @ B, and then I reshape the result with result.view(2, 3, 3, 3).
The resulting matrix is
tensor([[[[ 6687.5000,  7686.0000,  8684.5000],
          [11680.0000, 12678.5000, 13677.0000],
          [16672.5000, 17671.0000, 18669.5000]],

         [[ 6663.5000,  7658.0000,  8652.5000],
          [11636.0000, 12630.5000, 13625.0000],
          [16608.5000, 17603.0000, 18597.5000]],

         [[31650.0000, 32648.5000, 33647.0000],
          [36642.5000, 37641.0000, 38639.5000],
          [41635.0000, 42633.5000, 43632.0000]]],

        [[[31526.0000, 32520.5000, 33515.0000],
          [36498.5000, 37493.0000, 38487.5000],
          [41471.0000, 42465.5000, 43460.0000]],

         [[56612.5000, 57611.0000, 58609.5000],
          [61605.0000, 62603.5000, 63602.0000],
          [66597.5000, 67596.0000, 68594.5000]],

         [[56388.5000, 57383.0000, 58377.5000],
          [61361.0000, 62355.5000, 63350.0000],
          [66333.5000, 67328.0000, 68322.5000]]]], dtype=torch.float64)

Instead I want
tensor([[[[ 6687.5000,  7686.0000,  8684.5000],
          [11680.0000, 12678.5000, 13677.0000],
          [16672.5000, 17671.0000, 18669.5000]],

         [[31650.0000, 32648.5000, 33647.0000],
          [36642.5000, 37641.0000, 38639.5000],
          [41635.0000, 42633.5000, 43632.0000]],

         [[56612.5000, 57611.0000, 58609.5000],
          [61605.0000, 62603.5000, 63602.0000],
          [66597.5000, 67596.0000, 68594.5000]]],

        [[[ 6663.5000,  7658.0000,  8652.5000],
          [11636.0000, 12630.5000, 13625.0000],
          [16608.5000, 17603.0000, 18597.5000]],

         [[31526.0000, 32520.5000, 33515.0000],
          [36498.5000, 37493.0000, 38487.5000],
          [41471.0000, 42465.5000, 43460.0000]],

         [[56388.5000, 57383.0000, 58377.5000],
          [61361.0000, 62355.5000, 63350.0000],
          [66333.5000, 67328.0000, 68322.5000]]]], dtype=torch.float64)

Can someone help me? Thanks

Output:
This is a common but interesting problem because it involves a combination of torch.reshapes and torch.transpose to solve it. More specifically, you will need

Apply an initial reshape to restructure the tensor and expose the axes you want to swap;
Then do so using a transpose operation;
Lastly apply a second reshape to get to the desired format.

In your case, you could do:
>>> result.reshape(3,2,3,3).transpose(0,1).reshape(2,3,3,3)
tensor([[[[ 6687.5000,  7686.0000,  8684.5000],
          [11680.0000, 12678.5000, 13677.0000],
          [16672.5000, 17671.0000, 18669.5000]],

         [[31650.0000, 32648.5000, 33647.0000],
          [36642.5000, 37641.0000, 38639.5000],
          [41635.0000, 42633.5000, 43632.0000]],

         [[56612.5000, 57611.0000, 58609.5000],
          [61605.0000, 62603.5000, 63602.0000],
          [66597.5000, 67596.0000, 68594.5000]]],

        [[[ 6663.5000,  7658.0000,  8652.5000],
          [11636.0000, 12630.5000, 13625.0000],
          [16608.5000, 17603.0000, 18597.5000]],

         [[31526.0000, 32520.5000, 33515.0000],
          [36498.5000, 37493.0000, 38487.5000],
          [41471.0000, 42465.5000, 43460.0000]],

         [[56388.5000, 57383.0000, 58377.5000],
          [61361.0000, 62355.5000, 63350.0000],
          [66333.5000, 67328.0000, 68322.5000]]]], dtype=torch.float64)

I encourage you to look a the intermediate results to get an idea of how the method works so you can apply it on other use cases in the future.


Instruction: Pytorch TTS how to add output to audio file?

Input:
Is this code from this repo https://github.com/snakers4/silero-models#pytorch-1
I'd like to output the voice to an audio file, how can I do that?
code:
# V3
import torch

language = 'ru'
model_id = 'ru_v3'
sample_rate = 48000
speaker = 'ksenia'
device = torch.device('cpu')

model, example_text = torch.hub.load(repo_or_dir='snakers4/silero-models',
                                     model='silero_tts',
                                     language=language,
                                     speaker=model_id)
model.to(device)  # gpu or cpu

example_text = 'Привет, как делишки?'

audio = model.apply_tts(text=example_text,
                        speaker=speaker,
                        sample_rate=sample_rate)

I heard about this, but this don't work with v_2 or v_3
torchaudio.save('test_1.mp3',
                  audio[0].unsqueeze(0),
                  sample_rate=16000)

Output:
Just remove [0]
so this will be look like this:
torchaudio.save('test_1.mp3',
                  audio.unsqueeze(0),
                  sample_rate=16000)


Instruction: Symmetric random tensor with high dimension numpy/pytorch

Input:
I wish to create a random symmetric tensor such that for each permutation of indices i1, i2,...,ik i will have:
a[i1][i2]...[ik] = a[pi(i1)][pi(i2)]...[pi(ik)]

Where pi is a permutation function. For example:
a[1][2][3][4] = a[1][3][2][4] = a[4][2][1][3] = ...

In the 2 dim setting its the normal symmetry but i am unsure how implement this (without iterating over permutations) or if there is an automatic way of doing this using numpy / pytorch.

Output:
AFAIK there is no function for that in either torch nor numpy, but one might add all dimensionality permutated tensors to make it symmetric across every dimension.
This has:

O(size^dims) - memory complexity (as generators are used)
O(dims! * size^dims) - runtime complexity (although size^dims is vectorized and should be reasonably quick)

No idea how to improve runtime complexity though, sorry. Here is the code:
import functools
import itertools
import operator

import torch

def symmetricND(size: int, dims: int) -> torch.Tensor:
    data = torch.randn(*[size] * dims)
    return functools.reduce(
        operator.add,
        (
            torch.permute(data, permutation)
            for permutation in itertools.permutations(range(dims))
        ),
    )

And example usage (4x4x4 symmetrical tensor):
symmetric = symmetricND(4, 3)

print(symmetric[0][1][2])
print(symmetric[2][1][0])
print(symmetric[1][2][0])
print(symmetric[1][0][2])

In general: will run as long as you can hold single tensor in your memory, but might take a lot of time ((3, 7) is instant on my laptop, (3, 9)takes around 1 minute though, not sure if this is satisfactory for your case).
You could also precompute these tensors (or even use multiple machines for that), save them and load as needed.


Instruction: Pytorch expand class label tensor

Input:
I am working on a semantic segmentation project in pytorch and I have class maps in the following shapes: [H,W] where each element is an integer between 0-n where n is the number of classes, H the height of the image and W the width of the image.
Here is an example:
test_label = torch.zeros([10,10])
test_label[:5,:5] = 1
test_label[5:,:5] = 2
test_label[:5,5:] = 3
test_label

Output:
tensor([[1., 1., 1., 1., 1., 3., 3., 3., 3., 3.],
    [1., 1., 1., 1., 1., 3., 3., 3., 3., 3.],
    [1., 1., 1., 1., 1., 3., 3., 3., 3., 3.],
    [1., 1., 1., 1., 1., 3., 3., 3., 3., 3.],
    [1., 1., 1., 1., 1., 3., 3., 3., 3., 3.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.]])

Now, what I want is something of the shape [n,C,H] where [1,C,H] would be e.g.:
tensor([[1., 1., 1., 1., 1., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])

And [2,H,W] would be:
tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.],
    [2., 2., 2., 2., 2., 0., 0., 0., 0., 0.]])

Is there a pytorch function which does this? My current approach would be iteratively masking over each unique element in the original tensor and insert them into a tensor of the shape [n,H,W] initially filled with all zeros. But that doesn't seem to be the best way to do it. I tried to look it up but it seems like I am not able to find the right name for this operation.
Thank you very much for your time.

Output:
You could apply nn.functional.one_hot to convert the dense format into one-hot encoding and then multiple with the label value to get the desired result:
>>> C = int(x.max()) + 1
>>> ohe = F.one_hot(x.long(), num_classes=C)

Then multiple by the label values:
>>> res = ohe*torch.arange(C)
>>> res.permute(2,0,1)
tensor([[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],

        [[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
         [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
         [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
         [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
         [1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],

        [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0],
         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0],
         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0],
         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0],
         [2, 2, 2, 2, 2, 0, 0, 0, 0, 0]],

        [[0, 0, 0, 0, 0, 3, 3, 3, 3, 3],
         [0, 0, 0, 0, 0, 3, 3, 3, 3, 3],
         [0, 0, 0, 0, 0, 3, 3, 3, 3, 3],
         [0, 0, 0, 0, 0, 3, 3, 3, 3, 3],
         [0, 0, 0, 0, 0, 3, 3, 3, 3, 3],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]])


Instruction: how to save all the generated image in a folder in pytorch

Input:
I am trying to use data augmentation with pytorch. I want to save all the generated images in a folder (target_dir) with different numbering based on the batch index.
Here is my code. I am using epoch=100 and batch_size=128.
import os

for batch_idx in range(BATCH_SIZE):

torchvision.utils.save_image(img_grid_fake, f"C:/UserspythonProjectgenerated_image/Fake_image%{batch_idx}d.png", global_step=step)

but i am only getting last 128 generated images, previous generated image are get deleted when next epoch run.

Output:
You need to save the images with f"Fake_image-{epoch}-{batch-idx}.png" so that both epoch and batch_idx are used in naming the files.
import os
import torch
from torch import nn
import torchvision
from torch.utils.data import DataLoader
from torchvision.transforms import ToTensor

target_dir = r"C:/Users/PycharmProjects/pythonProject/generated/generated_image/"

EPOCHS = 10
BATCH_SIZE = 64
GRID_SIZE = 9 # 9 images in each grid
NUM_ROWS = 3 # sqrt(GRID_SIZE)

# if you want all the images in a batch to make the image-grid,
# set GRID_SIZE = BATCH_SIZE

train_dataset = YourFakeImageDataset()
train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE,
                              shuffle=True, transform=ToTensor())

for epoch in range(EPOCHS):
    for batch_idx, (X, y) in enumerate(train_dataloader):
        # assume X is the fake-image returned by the dataloader
        # and y is some target value for the X, also returned by the dataloader

        # ... do something with your images here
        # B, C, H, W = X.shape
        img_grid_fake = torchvision.utils.make_grid(X[:GRID_SIZE, ...], nrow=NUM_ROWS)
        filepath = os.path.join(target_dir, f"Fake_image-{epoch}-{batch_idx}.png")
        torchvision.utils.save_image(img_grid_fake, filepath)

NOTE: I cannot answer you properly, as your question does not specify a lot of details clearly (some of them are asked by others in the comments).
If you are making a fake-image-grid, how are you doing that? With torchvision.utils.make_grid()?
References

torchvision.utils.make_grid()
Visualizing a grid of images
torchvision.utils.make_grid()


Instruction: RuntimeError: shape '[32, 3, 224, 224]' is invalid for input of size 50176

Input:
Firstly, I have trained a model on 224,224,3 images and now I am working on visualization  taken from MNIST dataset codebase. Below code is worked fine on grayscale images but when i used for color images it didn't not work out.
Code Works fine
with torch.no_grad():
    while True:
        image = cv2.imread("example.png", flags=cv2.IMREAD_GRAYSCALE)
        print(image.shape)
        input_img_h, input_img_w = image.shape
        image = scale_transformation(image, scale_factor=scale_factors[scale_idx_factor])
        image = rotation_transformation(image, angle=rotation_factors[rotation_idx_factor])
        scale_idx_factor = (scale_idx_factor + 1) % len(scale_factors)
        rotation_idx_factor = (rotation_idx_factor + 1) % len(rotation_factors)

        image_tensor = torch.from_numpy(image) / 255.
        print("image_tensor.shape:", image_tensor.shape)

        image_tensor = image_tensor.view(1, 1, input_img_h, input_img_w)

        image_tensor = T.Normalize((0.1307,), (0.3081,))(image_tensor)
        image_tensor = image_tensor.to(device)

        out = model(image_tensor)

        image = np.repeat(image[..., np.newaxis], 3, axis=-1)
        roi_y, roi_x = input_img_h // 2, input_img_w // 2
        plot_offsets(image, save_output, roi_x=roi_x, roi_y=roi_y)

        save_output.clear()
        image = cv2.resize(image, dsize=(224, 224))
        cv2.imshow("image", image)
        key = cv2.waitKey(30)
        if key == 27:
            break

Code with problem: I have changed image size only
with torch.no_grad():
    while True:
        image = cv2.imread("image_06764.jpg")
        image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        print('Original Dimensions : ', image.shape)

        width = 224
        height = 224
        dim = (width, height)
        image = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
        # print(resized.shape[0])
        input_img_h = image.shape[0]
        input_img_w = image.shape[1]

        image = scale_transformation(image, scale_factor=scale_factors[scale_idx_factor])
        print("dfdf", image.shape)
        image = rotation_transformation(image, angle=rotation_factors[rotation_idx_factor])
        scale_idx_factor = (scale_idx_factor + 1) % len(scale_factors)
        rotation_idx_factor = (rotation_idx_factor + 1) % len(rotation_factors)

        image_tensor = torch.from_numpy(image) / 255.
        print("ggggggggggg", image_tensor.size())

        image_tensor = image_tensor.view(32, 3, input_img_h, input_img_w)
        print("image_tensor.shape:", image_tensor.shape)
        image_tensor = T.Normalize((0.1307,), (0.3081,))(image_tensor)
        image_tensor = image_tensor.to(device)
        out = model(image_tensor)

        image = np.repeat(image[..., np.newaxis], 3, axis=-1)
        roi_y, roi_x = input_img_h // 2, input_img_w // 2
        plot_offsets(image, save_output, roi_x=roi_x, roi_y=roi_y)

        save_output.clear()
        image = cv2.resize(image, dsize=(224, 224))
        cv2.imshow("image", image)
        key = cv2.waitKey(30)
        if key == 27:
            break

Traceback
Traceback (most recent call last):
  File "/media/cvpr/CM_1/tutorials/Deformable_Convolutionv_V2/offset_visualization.py", line 184, in <module>
    image_tensor = image_tensor.view(32, 3, input_img_h, input_img_w)
RuntimeError: shape '[32, 3, 224, 224]' is invalid for input of size 50176

Output:
image_tensor is a tensor size of 50176, which can be resized to 224x224. However, you're trying to resize it to 32x3x224x224.
Try this:
image_tensor = image_tensor.view(1, 1, input_img_h, input_img_w).repeat(1, 3, 1, 1)

Above code will copy the grayscale image 3 time channel-wise, resulting a tensor size of 1x3x224x224.
Additionally, why are you converting the color image to grayscale image with image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)? There will be no channel problem if you remove it.
Any advise or error correction of the answer is welcomed


Instruction: Pytorch matrix multiplication

Input:
I'm struggling with dimension and matric multiplication in pytorch.
I want to multiply matrix A
tensor([[[104.7500, 111.3750, 138.2500, 144.8750],
         [104.2500, 110.8750, 137.7500, 144.3750]],

        [[356.8750, 363.5000, 390.3750, 397.0000],
         [356.3750, 363.0000, 389.8750, 396.5000]]])

with matrix B
tensor([[[[  0.,   1.,   2.,   5.,   6.,   7.,  10.,  11.,  12.],
          [  2.,   3.,   4.,   7.,   8.,   9.,  12.,  13.,  14.],
          [ 10.,  11.,  12.,  15.,  16.,  17.,  20.,  21.,  22.],
          [ 12.,  13.,  14.,  17.,  18.,  19.,  22.,  23.,  24.]],

         [[ 25.,  26.,  27.,  30.,  31.,  32.,  35.,  36.,  37.],
          [ 27.,  28.,  29.,  32.,  33.,  34.,  37.,  38.,  39.],
          [ 35.,  36.,  37.,  40.,  41.,  42.,  45.,  46.,  47.],
          [ 37.,  38.,  39.,  42.,  43.,  44.,  47.,  48.,  49.]],

         [[ 50.,  51.,  52.,  55.,  56.,  57.,  60.,  61.,  62.],
          [ 52.,  53.,  54.,  57.,  58.,  59.,  62.,  63.,  64.],
          [ 60.,  61.,  62.,  65.,  66.,  67.,  70.,  71.,  72.],
          [ 62.,  63.,  64.,  67.,  68.,  69.,  72.,  73.,  74.]]],

        [[[ 75.,  76.,  77.,  80.,  81.,  82.,  85.,  86.,  87.],
          [ 77.,  78.,  79.,  82.,  83.,  84.,  87.,  88.,  89.],
          [ 85.,  86.,  87.,  90.,  91.,  92.,  95.,  96.,  97.],
          [ 87.,  88.,  89.,  92.,  93.,  94.,  97.,  98.,  99.]],

         [[100., 101., 102., 105., 106., 107., 110., 111., 112.],
          [102., 103., 104., 107., 108., 109., 112., 113., 114.],
          [110., 111., 112., 115., 116., 117., 120., 121., 122.],
          [112., 113., 114., 117., 118., 119., 122., 123., 124.]],

         [[125., 126., 127., 130., 131., 132., 135., 136., 137.],
          [127., 128., 129., 132., 133., 134., 137., 138., 139.],
          [135., 136., 137., 140., 141., 142., 145., 146., 147.],
          [137., 138., 139., 142., 143., 144., 147., 148., 149.]]]])

However using the simple @ to multiply them, doesn'e lead me to the desired result.
What I want is somethinlg like: multiply the first two rows of A by the first 3 4x9 submatrices of B (let's say B[:,:,0,:]) so that I have two results, then in the same way muliply the third and fourth row of A with the second 3 4x9 submatrices of B, so to have again two results, then I want to sum the first results of each multiplication and the second results of each.
I know I have to work with some kind of reshapes but I find it so confusing, can you help me with a quite generalizable solution?

Output:
This example would be helpful:
a = torch.ones((4, 4)).long()
a = a.reshape(2, 2, 4)
b = torch.tensor(list(range(36*6)))
b = b.reshape(2, 3, 4, 9)

t1 = a[0] @ b[0, :]
t2 = a[1] @ b[1, :]
result = t1 + t2

accum = torch.zeros((b.shape[1], a.shape[1], b.shape[3]))
for i in range(a.shape[0]):
  accum = accum + (a[i] @ b[i, :])


Instruction: How can I overcome PyTorch Tensor plotting problem?

Input:
I am a new PyTorch user and here is the code I am playing with.
epochs=20  # train for this number of epochs
losses=[] #to keep track on losses
for i in range(epochs):
    i+=1 #counter

    y_pred=model(cat_train,con_train)
    loss=torch.sqrt(criterion(y_pred,y_train))
    losses.append(loss)  # append loss values

    if i%10==1: # print out our progress
        print(f'epoch: {i} loss is {loss}')
    # back propagation
    optimizer.zero_grad() # find the zero gradient
    loss.backward() #move backward
    optimizer.step()
plt.plot(range(epochs),losses)

and it gives me the following error:
RuntimeError: Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy() instead.

I know the problem is related to the type of the losses with the following kind of rows:
tensor(3.6168, grad_fn=<SqrtBackward0>)

Can you suggest how I can grab the first column (numeric values of this tensor) and make it plottable e.i. an array not a Tensor.

Output:
You can use torch.Tensor.item.
So, replace the statement
losses.append(loss)

with
losses.append(loss.item())


Instruction: The problem of modified architecture of Resnet-101

Input:
Recently, I am implementing the ME-Net using pytorch, however, I encounter some problem.
In the ME-Net paper (as attached figure [1]: https://i.stack.imgur.com/1uKRc.png), the top part using ResNet-101 to extract features(using the pretrained ResNet-101 weight on the ImageNet datasets as the initial weight), whose kernel is arithmetic mean.
I use (256,256,11) as the image input, but i am not sure stage0~stage1 belongs to which part in ResNet-101 and can't find the output feature map size in each stage corresponding to the paper.
ME-Net paper:
ME-Net: A Deep Convolutional Neural Network for Extracting Mangrove Using Sentinel-2A Data
https://www.mdpi.com/2072-4292/13/7/1
from torchvision import models
res101 = models.resnet101(pretrained=True)

res101.conv1=nn.Conv2d(11, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
res101.cuda()
summary(res101, (11,256,256)

The summary of res101 :

    Layer (type)               Output Shape         Param #

        Conv2d-1         [-1, 64, 128, 128]          34,496
   BatchNorm2d-2         [-1, 64, 128, 128]             128
          ReLU-3         [-1, 64, 128, 128]               0
     MaxPool2d-4           [-1, 64, 64, 64]               0
        Conv2d-5           [-1, 64, 64, 64]           4,096
   BatchNorm2d-6           [-1, 64, 64, 64]             128
          ReLU-7           [-1, 64, 64, 64]               0
        Conv2d-8           [-1, 64, 64, 64]          36,864
   BatchNorm2d-9           [-1, 64, 64, 64]             128
         ReLU-10           [-1, 64, 64, 64]               0
       Conv2d-11          [-1, 256, 64, 64]          16,384
  BatchNorm2d-12          [-1, 256, 64, 64]             512
       Conv2d-13          [-1, 256, 64, 64]          16,384
  BatchNorm2d-14          [-1, 256, 64, 64]             512
         ReLU-15          [-1, 256, 64, 64]               0
   Bottleneck-16          [-1, 256, 64, 64]               0
       Conv2d-17           [-1, 64, 64, 64]          16,384
  BatchNorm2d-18           [-1, 64, 64, 64]             128
         ReLU-19           [-1, 64, 64, 64]               0
       Conv2d-20           [-1, 64, 64, 64]          36,864
  BatchNorm2d-21           [-1, 64, 64, 64]             128
         ReLU-22           [-1, 64, 64, 64]               0
       Conv2d-23          [-1, 256, 64, 64]          16,384
  BatchNorm2d-24          [-1, 256, 64, 64]             512
         ReLU-25          [-1, 256, 64, 64]               0
   Bottleneck-26          [-1, 256, 64, 64]               0
       Conv2d-27           [-1, 64, 64, 64]          16,384
  BatchNorm2d-28           [-1, 64, 64, 64]             128
         ReLU-29           [-1, 64, 64, 64]               0
       Conv2d-30           [-1, 64, 64, 64]          36,864
  BatchNorm2d-31           [-1, 64, 64, 64]             128
         ReLU-32           [-1, 64, 64, 64]               0
       Conv2d-33          [-1, 256, 64, 64]          16,384
  BatchNorm2d-34          [-1, 256, 64, 64]             512
         ReLU-35          [-1, 256, 64, 64]               0
   Bottleneck-36          [-1, 256, 64, 64]               0
       Conv2d-37          [-1, 128, 64, 64]          32,768
  BatchNorm2d-38          [-1, 128, 64, 64]             256
         ReLU-39          [-1, 128, 64, 64]               0
       Conv2d-40          [-1, 128, 32, 32]         147,456
  BatchNorm2d-41          [-1, 128, 32, 32]             256
         ReLU-42          [-1, 128, 32, 32]               0
       Conv2d-43          [-1, 512, 32, 32]          65,536
  BatchNorm2d-44          [-1, 512, 32, 32]           1,024
       Conv2d-45          [-1, 512, 32, 32]         131,072
  BatchNorm2d-46          [-1, 512, 32, 32]           1,024
         ReLU-47          [-1, 512, 32, 32]               0
   Bottleneck-48          [-1, 512, 32, 32]               0
       Conv2d-49          [-1, 128, 32, 32]          65,536
  BatchNorm2d-50          [-1, 128, 32, 32]             256
         ReLU-51          [-1, 128, 32, 32]               0
       Conv2d-52          [-1, 128, 32, 32]         147,456
  BatchNorm2d-53          [-1, 128, 32, 32]             256
         ReLU-54          [-1, 128, 32, 32]               0
       Conv2d-55          [-1, 512, 32, 32]          65,536
  BatchNorm2d-56          [-1, 512, 32, 32]           1,024
         ReLU-57          [-1, 512, 32, 32]               0
   Bottleneck-58          [-1, 512, 32, 32]               0
       Conv2d-59          [-1, 128, 32, 32]          65,536
  BatchNorm2d-60          [-1, 128, 32, 32]             256
         ReLU-61          [-1, 128, 32, 32]               0
       Conv2d-62          [-1, 128, 32, 32]         147,456
  BatchNorm2d-63          [-1, 128, 32, 32]             256
         ReLU-64          [-1, 128, 32, 32]               0
       Conv2d-65          [-1, 512, 32, 32]          65,536
  BatchNorm2d-66          [-1, 512, 32, 32]           1,024
         ReLU-67          [-1, 512, 32, 32]               0
   Bottleneck-68          [-1, 512, 32, 32]               0
       Conv2d-69          [-1, 128, 32, 32]          65,536
  BatchNorm2d-70          [-1, 128, 32, 32]             256
         ReLU-71          [-1, 128, 32, 32]               0
       Conv2d-72          [-1, 128, 32, 32]         147,456
  BatchNorm2d-73          [-1, 128, 32, 32]             256
         ReLU-74          [-1, 128, 32, 32]               0
       Conv2d-75          [-1, 512, 32, 32]          65,536
  BatchNorm2d-76          [-1, 512, 32, 32]           1,024
         ReLU-77          [-1, 512, 32, 32]               0
   Bottleneck-78          [-1, 512, 32, 32]               0
       Conv2d-79          [-1, 256, 32, 32]         131,072
  BatchNorm2d-80          [-1, 256, 32, 32]             512
         ReLU-81          [-1, 256, 32, 32]               0
       Conv2d-82          [-1, 256, 16, 16]         589,824
  BatchNorm2d-83          [-1, 256, 16, 16]             512
         ReLU-84          [-1, 256, 16, 16]               0
       Conv2d-85         [-1, 1024, 16, 16]         262,144
  BatchNorm2d-86         [-1, 1024, 16, 16]           2,048
       Conv2d-87         [-1, 1024, 16, 16]         524,288
  BatchNorm2d-88         [-1, 1024, 16, 16]           2,048
         ReLU-89         [-1, 1024, 16, 16]               0
   Bottleneck-90         [-1, 1024, 16, 16]               0
       Conv2d-91          [-1, 256, 16, 16]         262,144
  BatchNorm2d-92          [-1, 256, 16, 16]             512
         ReLU-93          [-1, 256, 16, 16]               0
       Conv2d-94          [-1, 256, 16, 16]         589,824
  BatchNorm2d-95          [-1, 256, 16, 16]             512
         ReLU-96          [-1, 256, 16, 16]               0
       Conv2d-97         [-1, 1024, 16, 16]         262,144
  BatchNorm2d-98         [-1, 1024, 16, 16]           2,048
         ReLU-99         [-1, 1024, 16, 16]               0
  Bottleneck-100         [-1, 1024, 16, 16]               0
      Conv2d-101          [-1, 256, 16, 16]         262,144
 BatchNorm2d-102          [-1, 256, 16, 16]             512
        ReLU-103          [-1, 256, 16, 16]               0
      Conv2d-104          [-1, 256, 16, 16]         589,824
 BatchNorm2d-105          [-1, 256, 16, 16]             512
        ReLU-106          [-1, 256, 16, 16]               0
      Conv2d-107         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-108         [-1, 1024, 16, 16]           2,048
        ReLU-109         [-1, 1024, 16, 16]               0
  Bottleneck-110         [-1, 1024, 16, 16]               0
      Conv2d-111          [-1, 256, 16, 16]         262,144
 BatchNorm2d-112          [-1, 256, 16, 16]             512
        ReLU-113          [-1, 256, 16, 16]               0
      Conv2d-114          [-1, 256, 16, 16]         589,824
 BatchNorm2d-115          [-1, 256, 16, 16]             512
        ReLU-116          [-1, 256, 16, 16]               0
      Conv2d-117         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-118         [-1, 1024, 16, 16]           2,048
        ReLU-119         [-1, 1024, 16, 16]               0
  Bottleneck-120         [-1, 1024, 16, 16]               0
      Conv2d-121          [-1, 256, 16, 16]         262,144
 BatchNorm2d-122          [-1, 256, 16, 16]             512
        ReLU-123          [-1, 256, 16, 16]               0
      Conv2d-124          [-1, 256, 16, 16]         589,824
 BatchNorm2d-125          [-1, 256, 16, 16]             512
        ReLU-126          [-1, 256, 16, 16]               0
      Conv2d-127         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-128         [-1, 1024, 16, 16]           2,048
        ReLU-129         [-1, 1024, 16, 16]               0
  Bottleneck-130         [-1, 1024, 16, 16]               0
      Conv2d-131          [-1, 256, 16, 16]         262,144
 BatchNorm2d-132          [-1, 256, 16, 16]             512
        ReLU-133          [-1, 256, 16, 16]               0
      Conv2d-134          [-1, 256, 16, 16]         589,824
 BatchNorm2d-135          [-1, 256, 16, 16]             512
        ReLU-136          [-1, 256, 16, 16]               0
      Conv2d-137         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-138         [-1, 1024, 16, 16]           2,048
        ReLU-139         [-1, 1024, 16, 16]               0
  Bottleneck-140         [-1, 1024, 16, 16]               0
      Conv2d-141          [-1, 256, 16, 16]         262,144
 BatchNorm2d-142          [-1, 256, 16, 16]             512
        ReLU-143          [-1, 256, 16, 16]               0
      Conv2d-144          [-1, 256, 16, 16]         589,824
 BatchNorm2d-145          [-1, 256, 16, 16]             512
        ReLU-146          [-1, 256, 16, 16]               0
      Conv2d-147         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-148         [-1, 1024, 16, 16]           2,048
        ReLU-149         [-1, 1024, 16, 16]               0
  Bottleneck-150         [-1, 1024, 16, 16]               0
      Conv2d-151          [-1, 256, 16, 16]         262,144
 BatchNorm2d-152          [-1, 256, 16, 16]             512
        ReLU-153          [-1, 256, 16, 16]               0
      Conv2d-154          [-1, 256, 16, 16]         589,824
 BatchNorm2d-155          [-1, 256, 16, 16]             512
        ReLU-156          [-1, 256, 16, 16]               0
      Conv2d-157         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-158         [-1, 1024, 16, 16]           2,048
        ReLU-159         [-1, 1024, 16, 16]               0
  Bottleneck-160         [-1, 1024, 16, 16]               0
      Conv2d-161          [-1, 256, 16, 16]         262,144
 BatchNorm2d-162          [-1, 256, 16, 16]             512
        ReLU-163          [-1, 256, 16, 16]               0
      Conv2d-164          [-1, 256, 16, 16]         589,824
 BatchNorm2d-165          [-1, 256, 16, 16]             512
        ReLU-166          [-1, 256, 16, 16]               0
      Conv2d-167         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-168         [-1, 1024, 16, 16]           2,048
        ReLU-169         [-1, 1024, 16, 16]               0
  Bottleneck-170         [-1, 1024, 16, 16]               0
      Conv2d-171          [-1, 256, 16, 16]         262,144
 BatchNorm2d-172          [-1, 256, 16, 16]             512
        ReLU-173          [-1, 256, 16, 16]               0
      Conv2d-174          [-1, 256, 16, 16]         589,824
 BatchNorm2d-175          [-1, 256, 16, 16]             512
        ReLU-176          [-1, 256, 16, 16]               0
      Conv2d-177         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-178         [-1, 1024, 16, 16]           2,048
        ReLU-179         [-1, 1024, 16, 16]               0
  Bottleneck-180         [-1, 1024, 16, 16]               0
      Conv2d-181          [-1, 256, 16, 16]         262,144
 BatchNorm2d-182          [-1, 256, 16, 16]             512
        ReLU-183          [-1, 256, 16, 16]               0
      Conv2d-184          [-1, 256, 16, 16]         589,824
 BatchNorm2d-185          [-1, 256, 16, 16]             512
        ReLU-186          [-1, 256, 16, 16]               0
      Conv2d-187         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-188         [-1, 1024, 16, 16]           2,048
        ReLU-189         [-1, 1024, 16, 16]               0
  Bottleneck-190         [-1, 1024, 16, 16]               0
      Conv2d-191          [-1, 256, 16, 16]         262,144
 BatchNorm2d-192          [-1, 256, 16, 16]             512
        ReLU-193          [-1, 256, 16, 16]               0
      Conv2d-194          [-1, 256, 16, 16]         589,824
 BatchNorm2d-195          [-1, 256, 16, 16]             512
        ReLU-196          [-1, 256, 16, 16]               0
      Conv2d-197         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-198         [-1, 1024, 16, 16]           2,048
        ReLU-199         [-1, 1024, 16, 16]               0
  Bottleneck-200         [-1, 1024, 16, 16]               0
      Conv2d-201          [-1, 256, 16, 16]         262,144
 BatchNorm2d-202          [-1, 256, 16, 16]             512
        ReLU-203          [-1, 256, 16, 16]               0
      Conv2d-204          [-1, 256, 16, 16]         589,824
 BatchNorm2d-205          [-1, 256, 16, 16]             512
        ReLU-206          [-1, 256, 16, 16]               0
      Conv2d-207         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-208         [-1, 1024, 16, 16]           2,048
        ReLU-209         [-1, 1024, 16, 16]               0
  Bottleneck-210         [-1, 1024, 16, 16]               0
      Conv2d-211          [-1, 256, 16, 16]         262,144
 BatchNorm2d-212          [-1, 256, 16, 16]             512
        ReLU-213          [-1, 256, 16, 16]               0
      Conv2d-214          [-1, 256, 16, 16]         589,824
 BatchNorm2d-215          [-1, 256, 16, 16]             512
        ReLU-216          [-1, 256, 16, 16]               0
      Conv2d-217         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-218         [-1, 1024, 16, 16]           2,048
        ReLU-219         [-1, 1024, 16, 16]               0
  Bottleneck-220         [-1, 1024, 16, 16]               0
      Conv2d-221          [-1, 256, 16, 16]         262,144
 BatchNorm2d-222          [-1, 256, 16, 16]             512
        ReLU-223          [-1, 256, 16, 16]               0
      Conv2d-224          [-1, 256, 16, 16]         589,824
 BatchNorm2d-225          [-1, 256, 16, 16]             512
        ReLU-226          [-1, 256, 16, 16]               0
      Conv2d-227         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-228         [-1, 1024, 16, 16]           2,048
        ReLU-229         [-1, 1024, 16, 16]               0
  Bottleneck-230         [-1, 1024, 16, 16]               0
      Conv2d-231          [-1, 256, 16, 16]         262,144
 BatchNorm2d-232          [-1, 256, 16, 16]             512
        ReLU-233          [-1, 256, 16, 16]               0
      Conv2d-234          [-1, 256, 16, 16]         589,824
 BatchNorm2d-235          [-1, 256, 16, 16]             512
        ReLU-236          [-1, 256, 16, 16]               0
      Conv2d-237         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-238         [-1, 1024, 16, 16]           2,048
        ReLU-239         [-1, 1024, 16, 16]               0
  Bottleneck-240         [-1, 1024, 16, 16]               0
      Conv2d-241          [-1, 256, 16, 16]         262,144
 BatchNorm2d-242          [-1, 256, 16, 16]             512
        ReLU-243          [-1, 256, 16, 16]               0
      Conv2d-244          [-1, 256, 16, 16]         589,824
 BatchNorm2d-245          [-1, 256, 16, 16]             512
        ReLU-246          [-1, 256, 16, 16]               0
      Conv2d-247         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-248         [-1, 1024, 16, 16]           2,048
        ReLU-249         [-1, 1024, 16, 16]               0
  Bottleneck-250         [-1, 1024, 16, 16]               0
      Conv2d-251          [-1, 256, 16, 16]         262,144
 BatchNorm2d-252          [-1, 256, 16, 16]             512
        ReLU-253          [-1, 256, 16, 16]               0
      Conv2d-254          [-1, 256, 16, 16]         589,824
 BatchNorm2d-255          [-1, 256, 16, 16]             512
        ReLU-256          [-1, 256, 16, 16]               0
      Conv2d-257         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-258         [-1, 1024, 16, 16]           2,048
        ReLU-259         [-1, 1024, 16, 16]               0
  Bottleneck-260         [-1, 1024, 16, 16]               0
      Conv2d-261          [-1, 256, 16, 16]         262,144
 BatchNorm2d-262          [-1, 256, 16, 16]             512
        ReLU-263          [-1, 256, 16, 16]               0
      Conv2d-264          [-1, 256, 16, 16]         589,824
 BatchNorm2d-265          [-1, 256, 16, 16]             512
        ReLU-266          [-1, 256, 16, 16]               0
      Conv2d-267         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-268         [-1, 1024, 16, 16]           2,048
        ReLU-269         [-1, 1024, 16, 16]               0
  Bottleneck-270         [-1, 1024, 16, 16]               0
      Conv2d-271          [-1, 256, 16, 16]         262,144
 BatchNorm2d-272          [-1, 256, 16, 16]             512
        ReLU-273          [-1, 256, 16, 16]               0
      Conv2d-274          [-1, 256, 16, 16]         589,824
 BatchNorm2d-275          [-1, 256, 16, 16]             512
        ReLU-276          [-1, 256, 16, 16]               0
      Conv2d-277         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-278         [-1, 1024, 16, 16]           2,048
        ReLU-279         [-1, 1024, 16, 16]               0
  Bottleneck-280         [-1, 1024, 16, 16]               0
      Conv2d-281          [-1, 256, 16, 16]         262,144
 BatchNorm2d-282          [-1, 256, 16, 16]             512
        ReLU-283          [-1, 256, 16, 16]               0
      Conv2d-284          [-1, 256, 16, 16]         589,824
 BatchNorm2d-285          [-1, 256, 16, 16]             512
        ReLU-286          [-1, 256, 16, 16]               0
      Conv2d-287         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-288         [-1, 1024, 16, 16]           2,048
        ReLU-289         [-1, 1024, 16, 16]               0
  Bottleneck-290         [-1, 1024, 16, 16]               0
      Conv2d-291          [-1, 256, 16, 16]         262,144
 BatchNorm2d-292          [-1, 256, 16, 16]             512
        ReLU-293          [-1, 256, 16, 16]               0
      Conv2d-294          [-1, 256, 16, 16]         589,824
 BatchNorm2d-295          [-1, 256, 16, 16]             512
        ReLU-296          [-1, 256, 16, 16]               0
      Conv2d-297         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-298         [-1, 1024, 16, 16]           2,048
        ReLU-299         [-1, 1024, 16, 16]               0
  Bottleneck-300         [-1, 1024, 16, 16]               0
      Conv2d-301          [-1, 256, 16, 16]         262,144
 BatchNorm2d-302          [-1, 256, 16, 16]             512
        ReLU-303          [-1, 256, 16, 16]               0
      Conv2d-304          [-1, 256, 16, 16]         589,824
 BatchNorm2d-305          [-1, 256, 16, 16]             512
        ReLU-306          [-1, 256, 16, 16]               0
      Conv2d-307         [-1, 1024, 16, 16]         262,144
 BatchNorm2d-308         [-1, 1024, 16, 16]           2,048
        ReLU-309         [-1, 1024, 16, 16]               0
  Bottleneck-310         [-1, 1024, 16, 16]               0
      Conv2d-311          [-1, 512, 16, 16]         524,288
 BatchNorm2d-312          [-1, 512, 16, 16]           1,024
        ReLU-313          [-1, 512, 16, 16]               0
      Conv2d-314            [-1, 512, 8, 8]       2,359,296
 BatchNorm2d-315            [-1, 512, 8, 8]           1,024
        ReLU-316            [-1, 512, 8, 8]               0
      Conv2d-317           [-1, 2048, 8, 8]       1,048,576
 BatchNorm2d-318           [-1, 2048, 8, 8]           4,096
      Conv2d-319           [-1, 2048, 8, 8]       2,097,152
 BatchNorm2d-320           [-1, 2048, 8, 8]           4,096
        ReLU-321           [-1, 2048, 8, 8]               0
  Bottleneck-322           [-1, 2048, 8, 8]               0
      Conv2d-323            [-1, 512, 8, 8]       1,048,576
 BatchNorm2d-324            [-1, 512, 8, 8]           1,024
        ReLU-325            [-1, 512, 8, 8]               0
      Conv2d-326            [-1, 512, 8, 8]       2,359,296
 BatchNorm2d-327            [-1, 512, 8, 8]           1,024
        ReLU-328            [-1, 512, 8, 8]               0
      Conv2d-329           [-1, 2048, 8, 8]       1,048,576
 BatchNorm2d-330           [-1, 2048, 8, 8]           4,096
        ReLU-331           [-1, 2048, 8, 8]               0
  Bottleneck-332           [-1, 2048, 8, 8]               0
      Conv2d-333            [-1, 512, 8, 8]       1,048,576
 BatchNorm2d-334            [-1, 512, 8, 8]           1,024
        ReLU-335            [-1, 512, 8, 8]               0
      Conv2d-336            [-1, 512, 8, 8]       2,359,296
 BatchNorm2d-337            [-1, 512, 8, 8]           1,024
        ReLU-338            [-1, 512, 8, 8]               0
      Conv2d-339           [-1, 2048, 8, 8]       1,048,576
 BatchNorm2d-340           [-1, 2048, 8, 8]           4,096
        ReLU-341           [-1, 2048, 8, 8]               0
  Bottleneck-342           [-1, 2048, 8, 8]               0
 AdaptiveAvgPool2d-343     [-1, 2048, 1, 1]               0
      Linear-344                 [-1, 1000]       2,049,000

I think the output feature map in stage0 is Layer:ReLU-3; the output feature map in stage1 is Layer:ReLU-32; the output feature map in stage2 is Layer:ReLU-81; the output feature map in stage3 is Layer:ReLU-313; I can't find the output feature maps in stage4 and stage5 in the size of (8,8,1024) and (4,4,2048).
-----Update below-----
To acquire the feature map in stage0~stage5, I use the code below:
stage5 = nn.Sequential(*list(res101.children())[:-2])
stage4 = nn.Sequential(*list(res101.children())[:-3])
stage3 = nn.Sequential(*list(res101.children())[:-4])
stage2 = nn.Sequential(*list(res101.children())[:-5])
stage1 = nn.Sequential(*list(res101.children())[:-6])
stage0 = nn.Sequential(*list(res101.children())[:-7])

However, the stage0~5 seems just represent the architecture.
How can I get their feature maps?

Output:
It is easier to check PyTorch’s source code for troubles like this one, see here.
Look at the ResNet class (which is used to create different variants via factory-like functions) for clues.
In this case, respective layers would be:

conv1 (stage0)
max pooled output of conv1 (stage1), this one is assumed by the shapes
layer1 (stage2)
layer2
layer3
layer4(stage5)

Also, you might use print(res101) to see all of the blocks for easier debugging as it has a hierarchical structure.
Obtaining features
You should use PyTorch FX for that (see here).
In your case it would be something along these lines:
from torchvision.models.feature_extractor import create_feature_extractor

extractor = create_feature_extractor(
    model,
    return_nodes=[
        ”conv1”,
        “maxpool”,
        “layer1”,
        “layer2”,
        “layer3”,
        “layer4”,
    ]
)

features = extractor(inputs)

Features should be a dict with keys being names of the layers specified above and values being resulting tensors.


Instruction: PyTorch ConvNet not working. Loss goes down as accuracy stays about %14

Input:
I am trying to learn pytorch and this is my first convolutional network. But the model is not training. Loss goes down on every epoch but accuracy fluctuates between 10-20%. I want to know what am i doing wrong to improve myself.
This is the data loading part
training_data = datasets.MNIST(
    root="data",
    train=True,
    download=True,
    transform=transforms.ToTensor(),
    target_transform=transforms.Lambda(lambda y: torch.zeros(10,dtype=torch.float).scatter_(0,torch.tensor(y),value=1))
)
test_data = datasets.MNIST(
    root="data",
    train=False,
    download=True,
    transform=transforms.ToTensor(),
    target_transform=transforms.Lambda(lambda y: torch.zeros(10,dtype=torch.float).scatter_(0,torch.tensor(y),value=1))
)
train_dataloader = DataLoader(training_data,batch_size=64,shuffle=True)
test_dataloader = DataLoader(test_data,batch_size=64,shuffle=True)

This is my model
from torch.nn.modules.pooling import MaxPool2d
class CNN(nn.Module):
  def __init__(self):
    super(CNN,self).__init__()
    self.CNN_stack = nn.Sequential(
        nn.ReflectionPad2d((1,0,1,0)),
        nn.Conv2d(in_channels=1,out_channels=5,kernel_size=5,stride=2),
        nn.ReLU(),
        nn.Conv2d(in_channels=5,out_channels=50,kernel_size=5,stride=2),
        nn.ReLU(),
        nn.Flatten(),
        nn.Linear(1250,100),
        nn.ReLU(),
        nn.Linear(100,10)
    )

  def forward(self,x):
    logits = self.CNN_stack(x)
    return logits

model = CNN().to(device)

These are my propagation loops
def train_loop(batch,X,y,model,loss_fn,optimizer):
  size = 60000

  #Forward Prop
  pred = model(X)
  loss = loss_fn(pred,y)

  #Backward Prop
  optimizer.zero_grad()
  loss.backward()
  optimizer.step()

  if batch % 100 == 0:
    loss, current = loss.item(), batch * len(X)
    print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

def test_loop(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for X, y in dataloader:
            X,y=X.to(device),y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(0) == y).type(torch.float).sum().item()
            #print(f"{pred[0].argmax(0)}={y[0]}")

    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),lr=learning_rate)

epochs = 10
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    for batch, (X,y) in enumerate(train_dataloader):
      X,y = X.to(device), y.to(device)
      train_loop(batch,X, y , model, loss_fn, optimizer)
    test_loop(test_dataloader , model, loss_fn)
print("Done!")

Output:
your accuracy calculation is not correct:

on pred side use argmax(1);
on y side note that y is one-hot encoded, so use argmax there, or something else.

this will work:
correct += (pred.argmax(1) == y.argmax(1)).sum().item()
Also use higher learning rate, like 0.01 to see faster learning.
With these changes your net yields Accuracy==97.6% after 10 epochs.