Instruction: Send and load an ML model over Apache Kafka

Input:
I've been looking around here and on the Internet, but it seems that I'm the first one having this question.
I'd like to train an ML model (let's say something with PyTorch) and write it to an Apache Kafka cluster. On the other side, there should be the possibility of loading the model again from the received array of bytes. It seems that almost all the frameworks only offer methods to load from a path, so a file.
The only constraint I'm trying to satisfy is to not save the model as a file, so I won't need a storage.
Am I missing something? Do you have any idea how to solve it?

Output:
One reason to avoid this is that Kafka messages have a default of 1MB max. Therefore sending models around in topics wouldn't be the best idea, and therefore why you could instead use model files, stored in a shared filesystem, and send URIs to the files (strings) to download in the consumer clients.
For small model files, there is nothing preventing you from dumping the Kafka record bytes to a local file, but if you happen to change the model input parameters, then you'd need to edit the consumer code, anyway.
Or you can embed the models in other stream processing engines (still on local filesystems), as linked in the comments.


Instruction: How to get time taken for each layer in Pytorch?

Input:
I want to know the inference time of a layer in Alexnet. This code measures the inference time of the first fully connected layer of Alexnet as the batch size changes. And I have a few questions about this.

Is it possible to measure the inference time accurately with the following code?
Is there a time difference because the CPU and GPU run separately?
Is there a module used to measure layer inference time in Pytorch?

Given the following code:
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms
import time
from tqdm import tqdm

class AlexNet(nn.Module):
    def __init__(self):
        super(AlexNet, self).__init__()

        self.relu = nn.ReLU(inplace=True)
        self.maxpool2D = nn.MaxPool2d(kernel_size=3, stride=2, padding=0)
        self.adaptive_avg_polling = nn.AdaptiveAvgPool2d((6, 6))
        self.dropout = nn.Dropout(p=0.5)

        self.conv1 = nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2)
        self.conv2 = nn.Conv2d(64, 192, kernel_size=5, padding=2)
        self.conv3 = nn.Conv2d(192, 384, kernel_size=3, padding=1)
        self.conv4 = nn.Conv2d(384, 256, kernel_size=3, padding=1)
        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(256 * 6 * 6, 4096)
        self.fc2 = nn.Linear(4096, 4096)
        self.fc3 = nn.Linear(4096, 1000)

    def time(self, x):
        x = self.maxpool2D(self.relu(self.conv1(x)))
        x = self.maxpool2D(self.relu(self.conv2(x)))
        x =                self.relu(self.conv3(x))
        x =                self.relu(self.conv4(x))
        x = self.maxpool2D(self.relu(self.conv5(x)))
        x = self.adaptive_avg_polling(x)

        x = x.view(x.size(0), -1)
        x = self.dropout(x)

        start1 = time.time()
        x = self.fc1(x)
        finish1 = time.time()

        x = self.dropout(self.relu(x))
        x = self.fc2(x)
        x = self.relu(x)
        x = self.fc3(x)

        return finish1 - start1

def layer_time():
     use_cuda = torch.cuda.is_available()
     print("use_cuda : ", use_cuda)

     FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
     device= torch.device("cuda:0" if use_cuda else "cpu")

     net = AlexNet().to(device)

     test_iter = 10000
     batch_size = 1
     for i in range(10):
         X = torch.randn(size=(batch_size, 3, 227, 227)).type(FloatTensor)
         s = 0.0
         for i in tqdm(range(test_iter)):
             s += net.time(X)
         print(s)
         batch_size *= 2

 layer_time()

Output:
I found a way to measure inference time by studying the AMP document. Using this, the GPU and CPU are synchronized and the inference time can be measured accurately.
import torch, time, gc

# Timing utilities
start_time = None

def start_timer():
    global start_time
    gc.collect()
    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.synchronize()
    start_time = time.time()

def end_timer():
    torch.cuda.synchronize()
    end_time = time.time()
    return end_time - start_time

So my code changes as follows:
import torch, time, gc
from tqdm import tqdm
import torch.nn as nn
import torch

# Timing utilities
start_time = None

def start_timer():
    global start_time
    gc.collect()
    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.synchronize()
    start_time = time.time()

def end_timer():
    torch.cuda.synchronize()
    end_time = time.time()
    return end_time - start_time

class AlexNet(nn.Module):
    def __init__(self):
        super(AlexNet, self).__init__()

        self.relu = nn.ReLU(inplace=True)
        self.maxpool2D = nn.MaxPool2d(kernel_size=3, stride=2, padding=0)
        self.adaptive_avg_polling = nn.AdaptiveAvgPool2d((6, 6))
        self.dropout = nn.Dropout(p=0.5)

        self.conv1 = nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2)
        self.conv2 = nn.Conv2d(64, 192, kernel_size=5, padding=2)
        self.conv3 = nn.Conv2d(192, 384, kernel_size=3, padding=1)
        self.conv4 = nn.Conv2d(384, 256, kernel_size=3, padding=1)
        self.conv5 = nn.Conv2d(256, 256, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(256 * 6 * 6, 4096)
        self.fc2 = nn.Linear(4096, 4096)
        self.fc3 = nn.Linear(4096, 1000)

    def time(self, x):
        x = self.maxpool2D(self.relu(self.conv1(x)))
        x = self.maxpool2D(self.relu(self.conv2(x)))
        x =                self.relu(self.conv3(x))
        x =                self.relu(self.conv4(x))
        x = self.maxpool2D(self.relu(self.conv5(x)))
        x = self.adaptive_avg_polling(x)

        x = x.view(x.size(0), -1)
        x = self.dropout(x)

        # Check first linear layer inference time
        start_timer()
        x = self.fc1(x)
        result = end_timer()

        x = self.dropout(self.relu(x))
        x = self.fc2(x)
        x = self.relu(x)
        x = self.fc3(x)

        return result

def layer_time():
     use_cuda = torch.cuda.is_available()
     print("use_cuda : ", use_cuda)

     FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
     device= torch.device("cuda:0" if use_cuda else "cpu")

     net = AlexNet().to(device)

     test_iter = 1000
     batch_size = 1
     for i in range(10):
         X = torch.randn(size=(batch_size, 3, 227, 227)).type(FloatTensor)
         s = 0.0
         for i in tqdm(range(test_iter)):
             s += net.time(X)
         print(s)
         batch_size *= 2

layer_time()


Instruction: Assigning custom weights to embedding layer in PyTorch

Input:
Does PyTorch's nn.Embedding support manually setting the embedding weights for only specific values?
I know I could set the weights of the entire embedding layer like this -
emb_layer = nn.Embedding(num_embeddings, embedding_dim)
emb_layer.weights = torch.nn.Parameter(torch.from_numpy(weight_matrix))

But does PyTorch provide any succinct/efficient method to set the embedding weights for only one particular value?
Something like emb_layer.set_weight(5) = torch.tensor([...]) to manually set the embedding only for the value "5"?

Output:
Yes. You can run emb_layer.weight.shape to see the shape of the weights, and then you can access and change a single weight like this, for example:
with torch.no_grad():
  emb_layer.weight[idx_1,idx_2] = some_value

I use two indices here since the embedding layer is two dimensional. Some layers, like a Linear layer, would only require one index.


Instruction: I this the correct way of computing the average accuracy?

Input:
I am fairly new to coding and getting confused between average accuracy and overall accuracy. I have created a function to calculate accuracy, i then divide this result by the len(dataloader) at the end of each epoch. Is this the correct way to calculate average accuracy?  If not could someone explain how I go about doing this correctly?
def accuracy(predictions, labels):
   classes = torch.argmax(predictions, dim=1)
   return torch.mean((classes == labels).float())

def train(model, optimizer, dataloader):

  #Setting model to train mode
  model.train()

  acc = 0.0
  loss = 0.0

  loss_fc = nn.CrossEntropyLoss()

  for i, (img, label) in enumerate(dataloader):

    #source images and labels to cpu device
    img, label = img.to(device), label.to(device)

    y_pred = model(img)

    optimizer.zero_grad()

    loss = loss_fc(y_pred, label)

    loss.backward()
    optimizer.step()

    #Update loss and accuracy
    loss += loss.item()
    acc += accuracy(y_pred, s_label)

  loss /= len(dataloader)
  acc /= len(dataloader)

Output:
Not sure what you mean by the overall and average accuracy. Typically accuracy is calculated at the end of each epoch. You pass the accuracy function your predictions and your actual labels and it returns what proportion you got right as a decimal (0-1).
I haven't seen any use for calculating the average accuracy across every epoch during training as this metric would be heavily impacted by how fast your model learns rather than how well it is able to eventually perform e.g. a model that needs a lot of epochs to do well will probably appear worse on this average accuracy than one that can converge on fewer epochs.
If you take a look at the accuracy score metric from scikit-learn it should help clear things up for you.
Link:
https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html
Hope this helps!


Instruction: Colab crashes when trying to create confusion matrix

Input:
I am trying to create a confusion matrix for my test set. My test set consists of 3585 images. Whenever I try to run the following code:
x_test,y_test = next(iter(dataloader)))
y_pred = resnet(x_test)

Google colab crashes using all the available RAM. Does anyone have a work around for this? Should I do this in batches?

Output:
Should I do this in batches?

Yes! Try to reduce batch size.
dataloader = ...  # reduce batch size here on dataloader creation
...

y_pred = []
for batch in dataloader:
    batch_y_pred = resnet(batch)
    y_pred.append(batch_y_pred)

I use list with append, you can try another way.


Instruction: Import error: 'SimpleExperiment' while running BOTORCH example code

Input:
I am trying to work with Bayesian Optimisation for my Numerical model run, Optimising its parameters. For this I am using BoTorch. Its example code is given as follows:
#!/usr/bin/env python3
# coding: utf-8

# ## Using a custom botorch model with Ax
#
# In this tutorial, we illustrate how to use a custom BoTorch model within Ax's `SimpleExperiment` API. This allows us to harness the convenience of Ax for running Bayesian Optimization loops, while at the same time maintaining full flexibility in terms of the modeling.
#
# Acquisition functions and strategies for optimizing acquisitions can be swapped out in much the same fashion. See for example the tutorial for [Implementing a custom acquisition function](./custom_acquisition).
#
# If you want to do something non-standard, or would like to have full insight into every aspect of the implementation, please see [this tutorial](./closed_loop_botorch_only) for how to write your own full optimization loop in BoTorch.

# ### Implementing the custom model
#
# For this tutorial, we implement a very simple gpytorch Exact GP Model that uses an RBF kernel (with ARD) and infers a (homoskedastic) noise level.
#
# Model definition is straightforward - here we implement a gpytorch `ExactGP` that also inherits from `GPyTorchModel` -- this adds all the api calls that botorch expects in its various modules.
#
# *Note:* botorch also allows implementing other custom models as long as they follow the minimal `Model` API. For more information, please see the [Model Documentation](../docs/models).

# In[1]:

from botorch.models.gpytorch import GPyTorchModel
from gpytorch.distributions import MultivariateNormal
from gpytorch.means import ConstantMean
from gpytorch.models import ExactGP
from gpytorch.kernels import RBFKernel, ScaleKernel
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.mlls import ExactMarginalLogLikelihood
from gpytorch.priors import GammaPrior

class SimpleCustomGP(ExactGP, GPyTorchModel):

    _num_outputs = 1  # to inform GPyTorchModel API

    def __init__(self, train_X, train_Y):
        # squeeze output dim before passing train_Y to ExactGP
        super().__init__(train_X, train_Y.squeeze(-1), GaussianLikelihood())
        self.mean_module = ConstantMean()
        self.covar_module = ScaleKernel(
            base_kernel=RBFKernel(ard_num_dims=train_X.shape[-1]),
        )
        self.to(train_X)  # make sure we're on the right device/dtype

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return MultivariateNormal(mean_x, covar_x)

# #### Define a factory function to be used with Ax's BotorchModel
#
# Ax's `BotorchModel` internally breaks down the different components of Bayesian Optimization (model generation & fitting, defining acquisition functions, and optimizing them) into a functional api.
#
# Depending on which of these components we want to modify, we can pass in an associated custom factory function to the `BotorchModel` constructor. In order to use a custom model, we have to implement a model factory function that, given data according to Ax's api specification, instantiates and fits a BoTorch Model object.
#
# The call signature of this factory function is the following:
#
# ```python
# def get_and_fit_gpytorch_model(
#     Xs: List[Tensor],
#     Ys: List[Tensor],
#     Yvars: List[Tensor],
#     state_dict: Optional[Dict[str, Tensor]] = None,
#     **kwargs: Any,
# ) -> Model:
# ```
#
# where
# - the `i`-th element of `Xs` are the training features for the i-th outcome as an `n_i x d` tensor (in our simple example, we only have one outcome)
# - similarly, the `i`-th element of `Ys` and `Yvars` are the observations and associated observation variances for the `i`-th outcome as `n_i x 1` tensors
# - `state_dict` is an optional PyTorch module state dict that can be used to initialize the model's parameters to pre-specified values
#
# The function must return a botorch `Model` object. What happens inside the function is up to you.
#
# Using botorch's `fit_gpytorch_model` utility function, model-fitting is straightforward for this simple model (you may have to use your own custom model fitting loop when working with more complex models - see the tutorial for [Fitting a model with torch.optim](fit_model_with_torch_optimizer).

# In[2]:

from botorch.fit import fit_gpytorch_model

def _get_and_fit_simple_custom_gp(Xs, Ys, **kwargs):
    model = SimpleCustomGP(Xs[0], Ys[0])
    mll = ExactMarginalLogLikelihood(model.likelihood, model)
    fit_gpytorch_model(mll)
    return model

# ### Set up the optimization problem in Ax

# Ax's `SimpleExperiment` API requires an evaluation function that is able to compute all the metrics required in the experiment. This function needs to accept a set of parameter values as a dictionary. It should produce a dictionary of metric names to tuples of mean and standard error for those metrics.
#
# For this tutorial, we use the Branin function, a simple synthetic benchmark function in two dimensions. In an actual application, this could be arbitrarily complicated - e.g. this function could run some costly simulation, conduct some A/B tests, or kick off some ML model training job with the given parameters).

# In[3]:

import random
import numpy as np

def branin(parameterization, *args):
    x1, x2 = parameterization["x1"], parameterization["x2"]
    y = (x2 - 5.1 / (4 * np.pi ** 2) * x1 ** 2 + 5 * x1 / np.pi - 6) ** 2
    y += 10 * (1 - 1 / (8 * np.pi)) * np.cos(x1) + 10
    # let's add some synthetic observation noise
    y += random.normalvariate(0, 0.1)
    return {"branin": (y, 0.0)}

# We need to define a search space for our experiment that defines the parameters and the set of feasible values.

# In[4]:

from ax import ParameterType, RangeParameter, SearchSpace

search_space = SearchSpace(
    parameters=[
        RangeParameter(
            name="x1", parameter_type=ParameterType.FLOAT, lower=-5, upper=10
        ),
        RangeParameter(
            name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=15
        ),
    ]
)

# Third, we make a `SimpleExperiment` — note that the `objective_name` needs to be one of the metric names returned by the evaluation function.

# In[5]:

from ax import SimpleExperiment

exp = SimpleExperiment(
    name="test_branin",
    search_space=search_space,
    evaluation_function=branin,
    objective_name="branin",
    minimize=True,
)

# We use the Sobol generator to create 5 (quasi-) random initial point in the search space. Calling `batch_trial` will cause Ax to evaluate the underlying `branin` function at the generated points, and automatically keep track of the results.

# In[6]:

from ax.modelbridge import get_sobol

sobol = get_sobol(exp.search_space)
exp.new_batch_trial(generator_run=sobol.gen(5))

# To run our custom botorch model inside the Ax optimization loop, we can use the `get_botorch` factory function from `ax.modelbridge.factory`. Any keyword arguments given to this function are passed through to the `BotorchModel` constructor. To use our custom model, we just need to pass our newly minted `_get_and_fit_simple_custom_gp` function to `get_botorch` using the `model_constructor` argument.
#
# **Note:** `get_botorch` by default automatically applies a number of parameter transformations (e.g. to normalize input data or standardize output data). This is typically what you want for standard use cases with continuous parameters. If your model expects raw parameters, make sure to pass in `transforms=[]` to avoid any transformations to take place. See the [Ax documentation](https://ax.dev/docs/models.html#transforms) for additional information on how transformations in Ax work.

# #### Run the optimization loop
#
# We're ready to run the Bayesian Optimization loop.

# In[7]:

from ax.modelbridge.factory import get_botorch

for i in range(5):
    print(f"Running optimization batch {i+1}/5...")
    model = get_botorch(
        experiment=exp,
        data=exp.eval(),
        search_space=exp.search_space,
        model_constructor=_get_and_fit_simple_custom_gp,
    )
    batch = exp.new_trial(generator_run=model.gen(1))

print("Done!")

I have installed BoTorch as :
conda install botorch -c pytorch -c gpytorch

Upon doing :
python3 custom_botorch_model_in_ax.py

I am getting the error message :
Traceback (most recent call last):
  File "/Users/sayantanmandal/Downloads/custom_botorch_model_in_ax.py", line 136, in <module>
    from ax import SimpleExperiment
ImportError: cannot import name 'SimpleExperiment' from 'ax' (/Users/sayantanmandal/opt/miniconda3/lib/python3.9/site-packages/ax/__init__.py)

There is some issue with some submodule SimpleExperiment from module ax.
Installing ax via:
conda or pip install ax, does not solve this issue. What steps should I take from here. Your help is greatly appreciated. Thanks.

Output:
Looks like SimpleExperiment was dropped starting with 0.2.4. An issue response recommends switching to Experiment. Otherwise, role back to 0.2.3.
If this example was found in version-controlled documentation, it would be nice if you notified the maintainers that the example needs updating.
Note that on Conda and PyPI, the ax module is delivered through the ax-platform package.


Instruction: pytorch gather failed with sparse_grad=True

Input:
With even very simple example, backward() cannot work if sparse_grad=True, please see the error below.
Is this error expected, or I'm using gather in a wrong way?
In [1]: import torch as th

In [2]: x = th.rand((3,3), requires_grad=True)

# sparse_grad = False, the backward could work as expetecd
In [3]: th.gather(x @ x, 1, th.LongTensor([[0], [1]]), sparse_grad=False).sum().backward()

# sparse_grad = True, backward CANNOT work
In [4]: th.gather(x @ x, 1, th.LongTensor([[0], [1]]), sparse_grad=True).sum().backward()
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
----> 1 th.gather(x @ x, 1, th.LongTensor([[0], [1]]), sparse_grad=True).sum().backward()

~/miniconda3/lib/python3.9/site-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    305                 create_graph=create_graph,
    306                 inputs=inputs)
--> 307         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    308
    309     def register_hook(self, hook):

~/miniconda3/lib/python3.9/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    152         retain_graph = create_graph
    153
--> 154     Variable._execution_engine.run_backward(
    155         tensors, grad_tensors_, retain_graph, create_graph, inputs,
    156         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag

RuntimeError: sparse tensors do not have strides

Output:
I think torch.gather does not support sparse operators:
torch.gather(x, 1, torch.LongTensor([[0], [1]]).to_sparse())

Results with:

NotImplementedError: Could not run 'aten::gather.out' with arguments from the 'SparseCPU' backend.

I think you should open an issue or a feature request on pytorch's github.


Instruction: How to iterate through all parameters in a neural network using pytorch

Input:
I have the following simple fully-connected neural network:
class Neural_net(nn.Module):
    def __init__(self):
        super(Neural_net, self).__init__()
        self.fc1    = nn.Linear(2, 2)
        self.fc2    = nn.Linear(2, 1)
        self.fc_out = nn.Linear(1, 1)

    def forward(self, x,train = True):
        x = torch.tanh(self.fc1(x))
        x = torch.tanh(self.fc2(x))
        x = self.fc_out(x)
        return x

net = Neural_net()

How can I loop through all the parameters of the network and check if for example they are greater than a certain value? I am using pytorch and if I do:
for n,p in net.named_parameters():
       if p > value:
       ...

I get an error since p is not a single number, but rather a tensor of either weights or biases for each layer.
My goal is to check if a criterion is satisfied for each of the parameters and flag them e.g. with 1 if it is or 0 if it is not, storing it in a dictionary with the same structure as net.parameters(). Yet, I am having trouble figuring out how to loop through them.
I thought about creating a parameter vector:
param_vec  =  torch.cat([p.view(-1) for p in net.parameters()])

and then accessing the parameter values and checking them would be easy,but then I can't think of a way to go back to the dictionary form  to flag them.
Thank you for any help!

Output:
First I would define the criterion as an operation on a tensor. In your case, this could look like this:
cond = lambda tensor: tensor.gt(value)

Then you just need to apply it to each tensor in net.parameters(). To keep it with the same structure, you can do it with dict comprehension:
cond_parameters = {n: cond(p) for n,p in net.named_parameters()}

Let's see it in practice!
net = Neural_net()
print(dict(net.parameters())
#> {'fc1.weight': Parameter containing:
#>  tensor([[-0.4767,  0.0771],
#>          [ 0.2874,  0.5474]], requires_grad=True),
#>  'fc1.bias': Parameter containing:
#>  tensor([ 0.0405, -0.1997], requires_grad=True),
#>  'fc2.weight': Parameter containing:
#>  tensor([[0.5400, 0.3241]], requires_grad=True),
#>  'fc2.bias': Parameter containing:
#>  tensor([-0.5306], requires_grad=True),
#>  'fc_out.weight': Parameter containing:
#>  tensor([[-0.9706]], requires_grad=True),
#>  'fc_out.bias': Parameter containing:
#> tensor([-0.4174], requires_grad=True)}

Let's set value to zero and get the dict of parameters:
value = 0
cond = lambda tensor: tensor.gt(value)
cond_parameters = {n: cond(p) for n,p in net.named_parameters()}
#>{'fc1.weight': tensor([[False,  True],
#>         [ True,  True]]),
#> 'fc1.bias': tensor([ True, False]),
#> 'fc2.weight': tensor([[True, True]]),
#> 'fc2.bias': tensor([False]),
#> 'fc_out.weight': tensor([[False]]),
#> 'fc_out.bias': tensor([False])}


Instruction: Missmatching target size in criterion

Input:
I'm trying to use pytorch in for the IMBD dataset, to predict the positive and negative reviews. When I get to the training state, the following error is given by the criterion function:
ValueError: Target size (torch.Size([64])) must be the same as input size (torch.Size([1136, 64, 1]))

After some research, I saw that the error is because the output of the model is returning a tensor of size [1136, 64, 1], and criterion is expecting only batch results.
Howerver, I don't know how to solve this error.
My code:
import torch
import spacy
import torch.nn as nn
from torchtext.legacy import data
import sys
import csv
import torch.optim as optim
import re
import nltk
from nltk.corpus import stopwords
from torchtext import vocab
from torchtext.legacy.data import Field
from torchtext.legacy import datasets
import pandas as pd
import re
import torch.nn.functional as F
import torch.nn as nn
import torch.optim as optim
import random

SEED = 1234

torch.manual_seed(SEED) # For reproducibility
torch.backends.cudnn.deterministic = True

import torch.nn.functional as F
import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):
        super().__init__()
        self.embedding = nn.Embedding(input_dim, embedding_dim)
        self.hidden_fc = nn.Linear(embedding_dim, hidden_dim)
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, text):
        #text = [sent len, batch size]
        embedded = self.embedding(text)

        h_1 = F.relu(self.hidden_fc(embedded))

        # assert torch.equal(output[-1,:,:], h_1.squeeze(0))
                # [batch size, output dim]
        return self.fc(h_1.squeeze(0))

def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """

    #round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds)) # 0.75 --> 1 0.4 --> 0
    correct = (rounded_preds == y).float() #convert into float for division
    acc = correct.sum() / len(correct)
    return acc

def train(model, iterator, optimizer, criterion):

    epoch_loss = 0
    epoch_acc = 0

    model.train() #Train mode is on

    for batch in iterator:

        optimizer.zero_grad() #Reset the gradients
        predictions = model(batch.text) ## forward propagation
        print(predictions.shape)
        loss = criterion(predictions, batch.label)
        acc = binary_accuracy(predictions, batch.label)
        loss.backward() ## backward propagation / calculate gradients
        optimizer.step() ## update parameters
        epoch_loss += loss.item()
        epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

def evaluate(model, iterator, criterion):

    epoch_loss = 0
    epoch_acc = 0

    model.eval() #Evaluation mode is on

    with torch.no_grad():

        for batch in iterator:
            predictions = model(batch.text).squeeze(1)
            loss = criterion(predictions, batch.label)
            acc = binary_accuracy(predictions, batch.label)
            epoch_loss += loss.item()
            epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

TEXT = data.Field(tokenize = 'spacy',
                  tokenizer_language = 'en_core_web_sm',
                  lower = True)

LABEL = data.LabelField(dtype = torch.float)

train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)  ## IMDB reviews dataset
train_data, valid_data = train_data.split(random_state = random.seed(SEED))

MAX_VOCAB_SIZE = 25_000

TEXT.build_vocab(train_data, max_size = MAX_VOCAB_SIZE) #Build the vocabulary using the top frequent 25K words
LABEL.build_vocab(train_data)
BATCH_SIZE = 64
train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    (train_data, valid_data, test_data),
    batch_size = BATCH_SIZE)

INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 256
OUTPUT_DIM = 1

model = MLP(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)

import torch.optim as optim

optimizer = optim.SGD(model.parameters(), lr=1e-3)
criterion = nn.BCEWithLogitsLoss()

N_EPOCHS = 5

best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):
    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')

Output:
To summarize your problem, you have reviews you want to classify as positive or negative. To do so you train an embedding space to map each word to a vector, then output a probability for each sentence and supervised with the corresponding label using a binary cross-entropy loss nn.BCELossWithLogits.
You current model is comprised of:

nn.Embedding: embeds each word in the sequence independently thus converting the input tensor shape from (seq_len, batch_size) to (seq_len, batch_size, embedding_dim). Where seq_len is the number of tokens in your input sequence.

nn.Linear layer reduces the dimensionality by projecting the features, the tensor shape is converted from (seq_len, batch_size, embedding_dim) to (seq_len, batch_size, hidden_dim).

A non-linearity layer is applied to the sequence of word vectors. Note how the structure of the sentence is retained. And finally, apply a second linear layer to map from (seq_len, batch_size, hidden_dim) to (seq_len, batch_size, output_dim). Still with the sentence structure (cf. the dim=0 with seq_len).

This is the reason why you are getting (1136, 64, 1) as the predictions shape: 1136 must be your sequence length, 64 is BATCH_SIZE, while 1 is OUTPUT_DIM.
Yet you are trying to classify each sequence as a whole, what you would need instead is a single tensor or scalar value per sentence, i.e. a shape of (1, 64, 1). This implies reducing the first dimension corresponding to the sequence dimension, to a single value.
A straightforward and easy way to reduce the dimension such that you can represent the whole sentence with a single vector is by applying an average pool to the sentence. The average vector of the words in each sentence should give you the sentiment of the positiveness/negativeness of the overall sentence. You can apply this operator before the final projection to remain in a relatively high dimension, either with nn.AdaptiveAvgPool1d with an output size of 1 or simply torch.Tensor.mean.
Here is a possible implementation with nn.AdaptiveAvgPool1d:
class MLP(nn.Module):
    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):
        super().__init__()
        self.embedding = nn.Embedding(input_dim, embedding_dim)
        self.hidden_fc = nn.Linear(embedding_dim, hidden_dim)
        self.avg = nn.AdaptiveAvgPool1d(1) # reduces (N, C, L_in) to (N, C, 1)
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, text):
        # (seq_len, batch_size) = (1136, 64)
        embedded = self.embedding(text)
        # (seq_len, batch_size, embedding_dim) = (1136, 64, 100)
        h_1 = F.relu(self.hidden_fc(embedded))
        # (seq_len, batch_size, hidden_dim) = (1136, 64, 256)
        avg = self.avg(h_1.permute(1,2,0))
        # (batch_size, hidden_dim, 1) = (64, 256, 1)
        out = self.fc(avg.squeeze(-1))
        # (batch_size, 1, 1) = (64, 1)
        return out

Or with torch.Tensor.mean:
class MLP(nn.Module):
    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):
        super().__init__()
        self.embedding = nn.Embedding(input_dim, embedding_dim)
        self.hidden_fc = nn.Linear(embedding_dim, hidden_dim)
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, text):
        # (seq_len, batch_size) = (1136, 64)
        embedded = self.embedding(text)
        # (seq_len, batch_size, embedding_dim) = (1136, 64, 100)
        h_1 = F.relu(self.hidden_fc(embedded))
        # (seq_len, batch_size, hidden_dim) = (1136, 64, 256)
        avg = h_1.mean(0)
        # (batch_size, hidden_dim, 1) = (64, 256)
        out = self.fc(avg)
        # (batch_size, 1, 1) = (64, 1)
        return out

Alternative methods involve using more sophisticated neural network layers such as recurrent neural network blocks (nn.RNN, nn.LSTM, nn.GRU)...


Instruction: Understanding an Einsum usage for graph convolution

Input:
I am reading the code for the spatial-temporal graph convolution operation here:
https://github.com/yysijie/st-gcn/blob/master/net/utils/tgcn.py and I'm having some difficulty understanding what is happening with an einsum operation. In particular
For x a tensor of shape (N, kernel_size, kc//kernel_size, t, v), where
kernel_size is typically 3, and lets say kc=64*kernel_size, t is the number of frames, say 64, and v the number of vertices, say 25. N is the batch size.
Now for a tensor A of shape (3, 25, 25) where each dimension is a filtering op on the graph vertices, an einsum is computed as:
x = torch.einsum('nkctv,kvw->nctw', (x, A))

I'm not sure how to interpret this expression. What I think it's saying is that for each batch element, for each channel c_i out of 64, sum each of the three matrices obtained by matrix multiplication of the (64, 25) feature map at that channel with the value of A[i]. Do I have this correct? The expression is a bit of a mouthful, and notation wise there seems to be a bit of a strange usage of kc as one variable name, but then decomposition of k as kernel size and c as the number of channels (192//3 = 64) in the expression for the einsum. Any insights appreciated.

Output:
Helps when you look closely at the notation:

nkctv for left side
kvw on the right side
nctw being the result

Missing from the result are:

k
v

These elements are summed together into a single value and squeezed, leaving us the resulting shape.
Something along the lines of (expanded shapes (added 1s) are broadcasted and sum per element):

left: (n, k, c, t, v, 1)
right: (1, k, 1, 1,  v, w)

Now it goes (l, r for left and right):

torch.mul(l, r)
torch.sum(l, r, dim=(1, 4))
squeeze any singular dimensions

It is pretty hard to get, hence Einstein's summation helps in terms of thinking about resulting shapes “mixed” with each other, at least for me.


Instruction: How to judge a torch.tensor dtype is int or not?

Input: I want to check a Tensor convert from an image is normalized or not, i.e. the dtype is int or float. Is there convenient way to achieve this goal? I do not want an enumerated condition check like a.dtype == torch.int or a.dtype == torch.int32 or a.dtype ==torch.uint8 .... . Or is there another way to check an image tensor normalized or not?

Output:
As pointed out by jasonharper, you can use torch.is_floating_point to check your tensor:
if torch.is_floating_point(my_tensor):
  # the tensor is "normalized"...
else:
  # the tensor is an integer and needs to be normalized...


Instruction: How to resize image tensors

Input:
The following is my code where I'm converting every image to PIL and then turning them into Pytorch tensors:
transform = transforms.Compose([transforms.PILToTensor()])

# choose the training and test datasets
train_data = os.listdir('data/training/')
testing_data = os.listdir('data/testing/')
train_tensors = []
test_tensors = []

for train_image in train_data:
    img = Image.open('data/training/' + train_image)
    train_tensors.append(transform(img))

for test_image in testing_data:
    img = Image.open('data/testing/' + test_image)
    test_tensors.append(transform(img))

# Print out some stats about the training and test data
print('Train data, number of images: ', len(train_data))
print('Test data, number of images: ', len(testing_data))

batch_size = 20

train_loader = DataLoader(train_tensors, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_tensors, batch_size=batch_size, shuffle=True)

# specify the image classes
classes = ['checked', 'unchecked', 'other']

# obtain one batch of training images
dataiter = iter(train_loader)
images, labels = dataiter.next()
images = images.numpy()

However, I am getting this error:
RuntimeError: stack expects each tensor to be equal size, but got [4, 66, 268] at entry 0 and [4, 88, 160] at entry 1

This is because my images are not resized prior to PIL -> Tensor. What is the correct way of resizing data images?

Output:
Try to utilize ImageFolder from torchvision, and assuming that images have diff size, you can use CenterCrop or RandomResizedCrop depending on your task. Check the Full list.
Here is an example:
train_dir = "data/training/"

train_dataset = datasets.ImageFolder(
    train_dir,
    transforms.Compose([
        transforms.RandomResizedCrop(img_size),  # image size int or tuple
        # Add more transforms here
        transforms.ToTensor(),  # convert to tensor at the end
]))

train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)


Instruction: Selecting entries of a pytorch tensor with another tensor

Input:
I have a tensor a with with float entries and torch.Size([64,2]) and I also have a tensor b with torch.Size([64]). The entries of b are only 0 or 1.
I would like to get a new tensor c with torch.Size([64]) such that c[i] == a[i,b[i]] for every index i. How can I do that?
My attempt
I tried with torch.gather but without success. The following code gives me RuntimeError: Index tensor must have the same number of dimensions as input tensor
import torch
a = torch.zeros([64,2])
b = torch.ones(64).long()
torch.gather(input=a, dim=1,index=b)

Any help will be highly appreciated!

Output:
You can perform this straight with an indexing of a on both dimensions:

On dimension=0: a "sequential" indexing using torch.arange.

On dimension=1: indexing using b.

All in all, this gives:
>>> a[torch.arange(len(a)), b]

Alternatively you can use torch.gather, the operation you are looking for is:
# c[i] == a[i,b[i]]

The provided gather operation when applied on dim=1 provides something like:
# c[i,j] == a[i,b[i,j]]

As you can see, we need to account for the difference in shapes between between a and b. To do so, you can unsqueeze a singleton dimension on b (annotated with the letter j above) such than #b=(64, 1), for instance with b.unsqueeze(-1) or b[...,None]:
>>> a.gather(dim=1, index=b[...,None]).flatten()


Instruction: FileNotFoundError in module easyocr when running exe file

Input:
I am trying to run exe file and scan picture with easyocr, but here's occurs the error. Could someone help me, please?
Traceback (most recent call last):
  File "threading.py", line 954, in _bootstrap_inner
  File "threading.py", line 892, in run
  File "ZhongDon.py", line 26, in ocr_simp
  File "easyocr\easyocr.py", line 200, in __init__
  File "easyocr\easyocr.py", line 261, in setLanguageList
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\Idensas\\AppData\\Local\\Temp\\_MEI44842\\easyocr\\character\\ch_sim_char.txt'

There are also warnings, but I ignored them
[4484] WARNING: file already exists but should not: C:\Users\Idensas\AppData\Local\Temp\_MEI44842\torch\_C.cp39-win_amd64.pyd
torch\_jit_internal.py:750: UserWarning: Unable to retrieve source for @torch.jit._overload function: <function _DenseLayer.forward at 0x0000015A402A1D30>.
  warnings.warn(f"Unable to retrieve source for @torch.jit._overload function: {func}.")
torch\_jit_internal.py:750: UserWarning: Unable to retrieve source for @torch.jit._overload function: <function _DenseLayer.forward at 0x0000015A402D8040>.
  warnings.warn(f"Unable to retrieve source for @torch.jit._overload function: {func}.")

Output:
This has solved my problem
pyinstaller -F ZhongDon.py --collect-all easyocr

Found the solution here


Instruction: Converting images to Pytorch tensors loses label data

Input:
I have a dataset of images. In the below code, I am trying to convert them to Pytorch tensors by first converting them to PIL images:
# choose the training and test datasets
train_data = os.listdir('data/training/')
testing_data = os.listdir('data/testing/')
train_tensors = []
test_tensors = []

# Print out some stats about the training and test data
print('Train data, number of images: ', len(train_data))
print('Test data, number of images: ', len(testing_data))

# The transformation call to resize images and transform them into Tensors
transform = transforms.Compose([
    transforms.RandomResizedCrop((120,120)),
    transforms.PILToTensor()
])

# Converting every train/test image to a PIL image and then to a Pytorch tensor
for train_image in train_data:
    img = Image.open('data/training/' + train_image)
    train_tensors.append(transform(img))

for test_image in testing_data:
    img = Image.open('data/testing/' + test_image)
    test_tensors.append(transform(img))

However, in this process, the labels are completely lost. This is the output of train_tensors
 [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255]],

        [[255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         ...,
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255]],

        [[255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 254, 254],
         ...,
         [254, 254, 255,  ..., 254, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255]],

        [[255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         ...,
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255],
         [255, 255, 255,  ..., 255, 255, 255]]], dtype=torch.uint8)

When I use this tensor in a dataloader and try to extract the labels, I get a too many values to unpack error.
train_loader = DataLoader(train_tensors, batch_size=batch_size, shuffle=True)

dataiter = iter(train_loader)
images, labels = dataiter.__next__() #

How can I maintain my label data?

Output:
You had appended only transformed images to train_tensors with no labels.
Therefore, dataiter.__next__() will give you only the tensor you'd appended, but at the same time you're requiring 2 variables images and labels.
Append label information to the list or request for only one element will solve this error.


Instruction: What is the alternative of pytorch module.register_parameter(name, param) in tensorflow?

Input:
I'm trying to convert some pytorch code to tensorflow. In the pytorch code they are adding some extra parameter on every module of a model using module.register_parameter(name, param). How can i covert this part of code on tensorflow?
Sample code on below:
for module_name, module in self.model.named_modules():
    module.register_parameter(name, new_parameter)

Output:
tf.Variable is the equivalent of nn.Parameter in PyTorch. tf.Variable is mainly used to store model parameters since their values are constantly updated during training.
To use a tensor as a new model parameter, you need to convert it to tf.Variable. You can check here how to create variables from tensors.
If you want to add a model parameter in TensorFlow inside the model itself, you could simply create a variable inside the model class and it will be automatically registered as a model parameter by TensorFlow.
If you want to add a tf.Variable externally to a model as a model parameter, you could manually add it to the trainable_weights attribute of tf.keras.layers.Layer by extending it like this -
model.layers[-1].trainable_weights.extend([new_parameter])


Instruction: What is the different for torchvision.models.resnet and torch.hub.load?

Input:
There are two method for using resnet of pytorch.
methods 1:
import torch
model = torch.hub.load('pytorch/vision:v0.10.0', 'resnet50', pretrained=True)
model.eval()

methods 2:
import torch
net = models.resnet50(pretrained=True)

Are they load the same model. If not what is the difference?

Output:
The only difference that there is between your models if you load them in that way it's the number of layers, since you're loading resnet18 with Torch Hub and resnet50 with Models (thus, also the pretrained weights). They behave differently, you can see more about that in this paper.
Torch Hub also lets you publish pretrained models in your repository, but since you're loading it from 'pytorch/vision:v0.10.0' (which is the same repository from which Models is loading the neural networks), there should be no difference between:
model = torch.hub.load('pytorch/vision', 'resnet18', pretrained=True)

and
model = models.resnet18(pretrained=True)


Instruction: Externally add a Variable to a model in TensorFlow

Input:
I have an existing TensorFlow model, and I want to add a new "parameter" (a tf.Variable) to the model's list of parameters (such that it's trainable) and add it externally to the model's list of parameters / computational graph.
One approach that I tried, is to append the new parameters to the model's list of trainable weights, something like this (here new_parameter is a tf.Variable) -
model.layers[-1].trainable_weights.extend([new_parameter])
model.compile(....)

But I'm not sure if that's the best way to go about it. In PyTorch, we have nn.Parameter instead of tf.Variable, and we have register_parameter using which we can register tensors as new parameter's to the model's list of parameters. Is there any equivalent of register_parameter in TensorFlow? Or some other way to achieve the same goal?

Output:
It is possible by custom dense layer too ‼
[ Sample ]:
import os
from os.path import exists

import tensorflow as tf
import tensorflow_io as tfio

import matplotlib.pyplot as plt
import numpy as np

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
None
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)
print(physical_devices)
print(config)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Variables
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
PATH = os.path.join('F:\\datasets\\downloads\\Actors\\train\\Pikaploy', '*.tif')
PATH_2 = os.path.join('F:\\datasets\\downloads\\Actors\\train\\Candidt Kibt', '*.tif')
files = tf.data.Dataset.list_files(PATH)
files_2 = tf.data.Dataset.list_files(PATH_2)

list_file = []
list_file_actual = []
list_label = []
list_label_actual = [ 'Pikaploy', 'Pikaploy', 'Pikaploy', 'Pikaploy', 'Pikaploy', 'Candidt Kibt', 'Candidt Kibt', 'Candidt Kibt', 'Candidt Kibt', 'Candidt Kibt' ]
for file in files.take(15):
    image = tf.io.read_file( file )
    image = tfio.experimental.image.decode_tiff(image, index=0)
    list_file_actual.append(image)
    image = tf.image.resize(image, [32,32], method='nearest')
    list_file.append(image)
    list_label.append(1)

for file in files_2.take(18):
    image = tf.io.read_file( file )
    image = tfio.experimental.image.decode_tiff(image, index=0)
    list_file_actual.append(image)
    image = tf.image.resize(image, [32,32], method='nearest')
    list_file.append(image)
    list_label.append(9)

checkpoint_path = "F:\\models\\checkpoint\\" + os.path.basename(__file__).split('.')[0] + "\\TF_DataSets_01.h5"
checkpoint_dir = os.path.dirname(checkpoint_path)
loggings = "F:\\models\\checkpoint\\" + os.path.basename(__file__).split('.')[0] + "\\loggings.log"

if not exists(checkpoint_dir) :
    os.mkdir(checkpoint_dir)
    print("Create directory: " + checkpoint_dir)

log_dir = checkpoint_dir

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Class / Function
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
class MyDenseLayer(tf.keras.layers.Layer):
    def __init__(self, num_outputs, num_add):
        super(MyDenseLayer, self).__init__()
        self.num_outputs = num_outputs
        self.num_add = num_add

    def build(self, input_shape):
        self.kernel = self.add_weight("kernel",
        shape=[int(input_shape[-1]),
        self.num_outputs])

    def call(self, inputs):
        temp = tf.add( inputs, self.num_add )
        temp = tf.matmul(temp, self.kernel)
        return temp

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
DataSet
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
list_file = tf.cast( list_file, dtype=tf.int64 )
list_file = tf.constant( list_file, shape=(33, 1, 32, 32, 4), dtype=tf.int64)
list_label = tf.cast( list_label, dtype=tf.int64 )
list_label = tf.constant( list_label, shape=(33, 1, 1, 1), dtype=tf.int64)

dataset = tf.data.Dataset.from_tensor_slices(( list_file, list_label ))

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Initialize
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model = tf.keras.models.Sequential([
    tf.keras.layers.InputLayer(input_shape=( 32, 32, 4 )),
    tf.keras.layers.Normalization(mean=3., variance=2.),
    tf.keras.layers.Normalization(mean=4., variance=6.),
    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
    tf.keras.layers.MaxPooling2D((2, 2)),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Reshape((128, 225)),
    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(96, return_sequences=True, return_state=False)),
    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(96)),
])

layer = MyDenseLayer(10, 5)

model.add(layer)
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(192, activation='relu'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.summary()

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Optimizer
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
optimizer = tf.keras.optimizers.Nadam(
    learning_rate=0.00001, beta_1=0.9, beta_2=0.999, epsilon=1e-07,
    name='Nadam'
)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Loss Fn
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
lossfn = tf.keras.losses.SparseCategoricalCrossentropy(
    from_logits=False,
    reduction=tf.keras.losses.Reduction.AUTO,
    name='sparse_categorical_crossentropy'
)

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Model Summary
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
model.compile(optimizer=optimizer, loss=lossfn, metrics=['accuracy'])

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: FileWriter
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
if exists(checkpoint_path) :
    model.load_weights(checkpoint_path)
    print("model load: " + checkpoint_path)
    input("Press Any Key!")

"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
: Training
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
history = model.fit( dataset, batch_size=100, epochs=50 )
model.save_weights(checkpoint_path)

plt.plot(history.history['loss'])
plt.show()
plt.close()

input("...")

[ Output ]:
Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #
=================================================================
 normalization (Normalizatio  (None, 32, 32, 4)        0
 n)

 normalization_1 (Normalizat  (None, 32, 32, 4)        0
 ion)

 conv2d (Conv2D)             (None, 30, 30, 32)        1184

 max_pooling2d (MaxPooling2D  (None, 15, 15, 32)       0
 )

 dense (Dense)               (None, 15, 15, 128)       4224

 reshape (Reshape)           (None, 128, 225)          0

 bidirectional (Bidirectiona  (None, 128, 192)         247296
 l)

 bidirectional_1 (Bidirectio  (None, 192)              221952
 nal)

 my_dense_layer (MyDenseLaye  (None, 10)               1920
 r) ***  custom layer added

 flatten (Flatten)           (None, 10)                0

 dense_1 (Dense)             (None, 192)               2112

 dense_2 (Dense)             (None, 10)                1930

=================================================================
Total params: 480,618
Trainable params: 480,618
Non-trainable params: 0
_________________________________________________________________


Instruction: An error while trying to implement a polynomial regression with Pytorch - Gradients are None after loss.backward()

Input:
I am trying to implement a custom polynomial regression using PyTorch but during the training procedure my implementation fails to calculate the gradients; i.e. the weights are always None even after the loss.backward() command. Below I give all the necessary details.
Step 1 I generate some data points with the following commands:
import numpy as np
import torch
import matplotlib.pyplot as plt
from torch.autograd import Function
import torch.nn as nn
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
seed_value = 42
np.random.seed(seed_value)
x = np.sort(np.random.rand(1000))
y = np.cos(1.2 * x * np.pi) + (0.1 * np.random.randn(1000))

and then I use train-test split from sklearn to split my data into training and test sets.
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(x,y,train_size = 0.7,
                                                   random_state = seed_value)

Step 2 I create the custom function named poly which returns the value of the polynomial p(x)=w0+w1x+...w5x^5, evaluated at x for given weights w.
def poly(x,w,batch_size = 10,degree = 5):
    x = x.repeat(1,degree+1)
    w = w.repeat(batch_size,1)
    exp = torch.arange(0.,degree+1).repeat(batch_size,1)
    return torch.sum(w*torch.pow(x,exp),dim=1)

Step 3 I construct the class custom_dataset which inherits from PyTorch's dataset to handle my training into batches.
class custom_dataset(Dataset):
    def __init__(self,X,y):
        self.x = torch.from_numpy(X).type(torch.float32).reshape(len(X),1)
        self.y = torch.from_numpy(y).type(torch.float32)
    def __len__(self):
        return len(self.x)
    def __getitem__(self,idx):
        return self.x[idx], self.y[idx]

Step 4 I construct the loop handling the training procedure.
training_data = custom_dataset(X_train,y_train)
test_data = custom_dataset(X_test,y_test)
def training_loop(train_loader, w, epochs, lr, batch_size,
                  loss_fn = nn.MSELoss(), degree = 5):
    weights = torch.tensor(w,dtype = torch.float32, requires_grad = True)
    num_batches = len(train_loader)//batch_size
    for epoch in range(1,epochs+1):
        print(f"{5*'-'}>epoch:{epoch}<{5*'-'}")
        for i,sample in enumerate(train_loader):
            x,y = sample
            y_preds = poly(x,weights,batch_size = batch_size)
            loss = loss_fn(y,y_preds)
            loss.backward() # backpropagation
            weights = weights - lr*weights.grad # update - gradient descent

            if (i+1) % 100 == 0:
                print(f"- Batch:[{i+1}|{num_batches}]{5*' '}Samples:[{(i+1)*num_batches}|{len(train_loader)}]{5*' '}loss:{loss.item():.6f}")
    return w

Step 5 I start training...
epochs = 10
lr = 1e-3
batch_size = 10
degree = 5
train_loader = DataLoader(training_data, batch_size = batch_size,
                         shuffle = True)
test_loader = DataLoader(test_data, batch_size = batch_size,
                        shuffle = True)
w = [0]*(degree+1)
w = training_loop(train_loader, w = w, epochs = 30, lr = lr,
                  batch_size = batch_size)

and getting the following error
---------------------------------------------------------------------------  TypeError                                 Traceback (most recent call last) Input In [40], in <cell line: 10>()
      7 test_loader = DataLoader(test_data, batch_size = batch_size,
      8                         shuffle = True)
      9 w = [0]*(degree+1)
---> 10 w = training_loop(train_loader, w = w, epochs = 30, lr = lr,
     11                   batch_size = batch_size)

Input In [39], in training_loop(train_loader, w, epochs, lr, batch_size, loss_fn, degree)
     10 loss = loss_fn(y,y_preds)
     11 loss.backward() # backpropagation
---> 12 weights = weights - lr*weights.grad # update - gradient descent
     14 if (i+1) % 100 == 0:
     15     print(f"- Batch:[{i+1}|{num_batches}{5*' '}Samples:[{(i+1)*num_batches}|{len(train_loader)}]{5*' '}loss:{loss.item():.6f}")

TypeError: unsupported operand type(s) for *: 'float' and 'NoneType'

Which means that the computation of the gradients did not affect the variable weights as it still set to None. Do you have any idea what is wrong?

Output:
You are overwriting the weights variable on your first loop iteration which will be replaced with a copy of weights without the grad attribute. This behavior can be reproduced with the following minimal code:
>>> weights.grad =  torch.ones_like(weights)
>>> for i in range(2):
...     print(weights.grad)
...     weights = weights - weights.grad

tensor([1., 1.])
None

To fix this, you can update the parameter using an in-place operation:
        weights -= lr*weights.grad # update - gradient descent


Instruction: MSELoss from Pytorch

Input:
I am trying to train a neural network using Pytorch. I would like the loss function to be the MSE. I tried to use torch.nn.MSELoss, however I get an error that I do not understand.
For example the following code gives me RuntimeError: Boolean value of Tensor with more than one value is ambiguous
import torch
import torch.nn as nn

model = torch.zeros(64)
model.requires_grad = True
target = torch.ones(64)

loss = nn.MSELoss(model, target)

Any help will be very much appreciated!

Output:
Please look in Pytorch docs: https://pytorch.org/docs/stable/generated/torch.nn.MSELoss.html
You need to create an MSELoss object before calling with the target and predictions.
loss = nn.MSELoss()
input = torch.zeros(64, requires_grad=True)
target = torch.ones(64)
output = loss(input, target)


Instruction: Is this custom PyTorch loss function differentiable

Input:
I have a custom forward implementation for a PyTorch loss. The training works well. I've checked the loss.grad_fn and it is not None.
I'm trying to understand two things:

How this function can be differentiable since there is an if-else statement on the path from input to output?

Does the path from gt (ground truth input) to loss (output) need to be differentiable? or only the path from pred (prediction input)?

Here is the source code:
class FocalLoss(nn.Module):
    def __init__(self):
        super(FocalLoss, self).__init__()

    def forward(self, pred, gt):
        pos_inds = gt.eq(1).float()
        neg_inds = gt.lt(1).float()
        neg_weights = torch.pow(1 - gt, 4)

        pos_loss = torch.log(pred) * torch.pow(1 - pred, 2) * pos_inds
        neg_loss = torch.log(1 - pred) * torch.pow(pred, 2) * neg_weights * neg_inds

        num_pos = pos_inds.float().sum()
        pos_loss_s = pos_loss.sum()
        neg_loss_s = neg_loss.sum()
        if num_pos == 0:
            loss = - neg_loss_s
        else:
            loss = - (pos_loss_s + neg_loss_s) / num_pos

        return loss

Output:
The if statement is not part of the computational graph. It is part of the code used to build this graph dynamically (i.e. the forward function) but it isn't in itself part of it. The principle to follow is to ask yourself whether you backtrack to the leaves of the graph (tensors that do not have parents in the graph, i.e. inputs, and parameters) using grad_fn callbacks of each node, backpropagating through the graph. The answer is you can only do so if each of the operators is differentiable: in programming terms, they implement a backward function operation (a.k.a. grad_fn).

In your example, whether num_pos is equal to 0 or not, the resulting loss tensor will depend on neg_loss_s alone or on pos_loss_s and neg_loss_s. However in either cases, the resulting loss tensor remains attached to the input pred:

via one way: the "neg_loss_s" node
or the other: the "pos_loss_s" and "neg_loss_s" nodes.

In your setup, either way, the operation is differentiable.

If gt is a ground-truth tensor then it doesn't require gradient and the operation from it to the final loss doesn't need to be differentiable. This is the case in your example where both pos_inds, and neg_inds are non-differientblae because they are boolean operators.


Instruction: Understanding broadcasting and arithmetic operations on different dimension tensors

Input:
I'm currently working on computing various similarity metrics between vectors such as cosine similarity, euclidean distance, mahalanobis distance, etc. As I'm working with vectors that can be very large, I need compute time to be minimal.
I'm struggling to understand how to work with vectors of different dimensions (they, do, however, share one dimension) and how to work with this in PyTorch.
I have two vectors, A and B with dimensions [867, 768] and [621, 768], respectively.
I am trying to compute the following:

For each v_a of the 867 vectors in A,
Subtract v_a - v_b for each of the 621 vectors in B

I'm aware that this is achieveable under the hood with the likes of scipy and numpy but I'm trying to avoid detaching and moving the vectors to the CPU for speed.
Can someone help me understand the logic of the operators required in PyTorch to achieve this?

Output:
You could use fancy indexing on both input tensors to unsqueeze a dimension such that A and B have a shape of (1, 867, 768) and (621, 1, 768) respectively. The subtraction operation will then automatically broadcast the two tensors to identical shapes.
>>> diff = A[None]-B[:,None]
>>> diff.shape
torch.Size([621, 867, 768])

This is the typical approach when implementing batched-pairwise distances.
More specifically, notice the difference between the two variants: A[None]-B[:,None] and A[:,None]-B[None].
diff = A[None]-B[:,None]    # (1, k_a, b) - (k_b, 1, b) -> (k_b, k_a, b) - (k_b, k_a, b)
diff.shape                  # (k_b, k_a, b)

Compared to:
diff = A[:,None]-B[None]    # (k_a, 1, b) - (1, k_b, b) -> (k_a, k_b, b) - (k_a, k_b, b)
diff.shape                  # (k_a, k_b, b)

You can read more about broadcasting on the NumPy documentation page.


Instruction: Deploying Pytorch only for prediction

Input: I've trained my model locally and now I want to use it in my Kubernetes cluster. Unfortunately, all the Docker images for Pytorch are 5+ GBs because they contain the scripts for training which I won't need now. I've created my own image which is only 3.5 GBs but still huge. Is there a slim Pytorch version for predictions? If not, which parts of the package can I safely remove and how?

Output:
No easy answer for Python version of PyTorch unfortunately (or at least none I’m aware of).
Python, in general, is not well-suited for Docker deployments as it carries over the dependencies (even if you don't need all of their functionality, imports are often at the top of the file making your aforementioned removal infeasible for projects of PyTorch size and complexity).
There is a way out though...
torchscript
Given your trained model you can convert it to traced/scripted version (see here). After you manage that:
Inference in other languages
Write your inference code in another language, either Java or C++(see here for more info).
I have only used C++, but you might get there easier with Java, I think.
Results
Managed to get PyTorch for CPU inference to roughly ~32MB, GPU would weight more and be way more complex though and would probably need ~1GB of CUDNN dependency itself.
C++ way
Please note torchlambda project is not currently maintained and I’m the creator, hopefully it gives you some tips at least.
See:

Dockerfile for the image build
CMake used for building
Docs for more info about compilation options etc.
C++ inference code

Additional notes:

It also uses AWS SDKs and you would have to remove them from at least these files
You don't need static compilation - it would help to reach the lowest possible (I could come up with) image size, but not strictly necessary (additional ‘100MB’ or so)

Final

Try Java first as it’s packaging is probably saner (although final image would probably be a little bigger)
The C++ way not tested for the newest PyTorch version and might be subject to change with basically any release
In general it takes A LOT of time and debugging, unfortunately.


Instruction: Concatenate torch tensors

Input:
I have two tensors in PyTorch as:
a.shape, b.shape
# (torch.Size([512, 28, 2]), torch.Size([512, 28, 26]))

My goal is to join/merge/concatenate them together so that I get the shape: (512, 28, 28).
I tried:
torch.stack((a, b), dim = 2).shape
torch.cat((a, b)).shape

But none of them seem to work.
I am using PyTorch version: 1.11.0 and Python 3.9.
Help?

Output:
Set dim parameter to 2 to concatenate over last dimension:
a = torch.randn(512, 28, 2)
b = torch.randn(512, 28, 26)

print(a.size(), b.size())

# set dim=2 to concat over 2nd dimension
c = torch.cat((a, b), dim=2)

print(c.size())

torch.Size([512, 28, 2]) torch.Size([512, 28, 26])
torch.Size([512, 28, 28])


Instruction: Is it possible to auto-size the subsequent input of a layer following torch.nn.Flatten within torch.nn.Sequential in PyTorch?

Input:
If I have the following model class for example:
class MyTestModel(nn.Module):

    def __init__(self):
        super(MyTestModel, self).__init__()

        self.seq1 = nn.Sequential(
            nn.Conv2d(3, 6, 3),
            nn.MaxPool2d(2, 2),
            nn.Conv2d(6, 16, 3),
            nn.MaxPool2d(2, 2),
            nn.Flatten(),
            nn.Linear(myflattendinput(), 120), # how to automate this?
            nn.ReLU(),
            nn.Linear(120, 84),
            nn.ReLU(),
            nn.Linear(84, 2),
        )
        self.softmax = nn.Softmax(dim=1)

    def forward(self, x):

        x = self.seq1(x)
        x = self.softmax(x)
        return x

I know, normally you would let the data loader give a fixed size input to the model, thus having a fixed size for the input of the layer after nn.Flatten(), however I was wondering if you could somehow compute this automatically?

Output: PyTorch (>=1.8) has LazyLinear which infers the input dimension.


Instruction: Tensor shape for multivariable LSTM on Pytorch

Input:
I have a dataset with 8 features and 4 timesteps. I am trying to implement an LSTM but need help understanding if i have set my tensor correctly. The aim is to take the outputted features from the LSTM and pass them through a NN.
My tensor shape is currently #samples x #timesteps x #features i.e. 4500x4x8. This works with the code below. I want to make sure that the model is indeed taking each timestep matrix as a new sequence (with matrix 4500x[0]x8 being the first timestep matrix and 4500x[3]x8 being the last timestep). I then take the final timestep output (output[:,-1,:] to feed through a NN.
Is the code doing what i think it is doing? I ask as performance is marginally less than a simple RF that only uses the final timestep data. This would be unexpected as the data has strong time-series correlations (it tracks patients vitals declining before going on ventilation).
I have the following code:
class LSTM1(nn.Module):
    def __init__(self, num_classes, input_size, hidden_size, num_layers):
        super(LSTM1, self).__init__()
        self.num_classes = num_classes #number of classes
        self.num_layers = num_layers #number of layers
        self.input_size = input_size #input size
        self.hidden_size = hidden_size #hidden state

        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,
                          num_layers=num_layers, batch_first=True) #lstm
        self.fc_1 =  nn.Linear(hidden_size, 32) #fully connected 1
        self.fc_2 =  nn.Linear(32, 12) #fully connected 1
        self.fc_3 = nn.Linear(12, 1)
        self.fc = nn.Sigmoid() #fully connected last layer

        self.relu = nn.ReLU()
    def forward(self,x):
            h_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #hidden state
            c_0 = Variable(torch.zeros(self.num_layers, x.size(0), self.hidden_size)) #internal state
            # Propagate input through LSTM
            output, (hn, cn) = self.lstm(x, (h_0, c_0)) #lstm with input, hidden, and internal state
            out = output[:,-1,:] #reshaping the data for Dense layer next
            out = self.relu(out)
            out = self.fc_1(out) #first Dense
            out = self.relu(out) #relu
            out = self.fc_2(out) #2nd dense
            out = self.relu(out) #relu
            out = self.fc_3(out) #3rd dense
            out = self.relu(out) #relu
            out = self.fc(out) #Final Output
            return out

Output:
Error
Your error stems from the last three lines.

Do not use ReLU activation at the end of your network

Use nn.Linear -> nn.Sigmoid with BCELoss or
nn.Linear with nn.BCEWithLogitsLoss (see here for what logits are).
What is going on

With ReLu you output values in the range [0, +inf)
Applying sigmoid on top of it “squashes” values to (0, 1) with threshold being 0 (e.g. 0 becomes 0.5 probability, hence 1 after threaholding at 0.5!)
In effect, you always predict 1 with this code, which is not what you want probably


Instruction: Integer partition N into M parts in parallel

Input:
I am trying to randomly generate integer partitions (N into M parts) in pytorch with a minimum partition size of 1.
For example, (3, 1, 1) and (4, 1, 0) are both partitions of 5 into 3 parts but (4, 1, 0)'s minimum partition size is 0 so this should not be allowed
I would like to use this to generate my dataset on demand, so would nice if there was a pytorch (parrallel/gpgpu) solution.
See other questions about generating integer partitions:

Generate restricted weak integer compositions (or partitions) of an integer n into k parts in Python
Elegant Python code for Integer Partitioning

Output:
Here's a solution that works in limited cases, M hast to divide N, 2 has to divide M, and the maximium is limited, but this is the behaviour I wanted.
You start of with the equal partition then calculate a delta that sums to zero...
M = 4
N = 16
MINIMUM = 1
assert N % M == 0
assert M % 2 == 0
avg = N // M

equal_partition = torch.full((M,), avg)
half_delta = torch.randint(-(avg-MINIMUM), avg-MINIMUM, (M // 2,))
delta = torch.cat((half_delta, -half_delta), dim=-1)[torch.randperm(M)]
partition = equal_partition + delta
print(partition, partition.sum())


Instruction: Failed to use transforms.ToTensor and transforms.Normalize to normalize the MNIST dataset

Input:
I used the following code to normalize the MNIST dataset, when I print the first sample, it fails to normalize as the max element is 255, not 1.
train_transform = transforms.Compose([
   transforms.ToTensor(),
   transforms.Normalize((0.1307,), (0.3081,))])

train_set = torchvision.datasets.MNIST(
   root=data_dir, train=True, download=True, transform=train_transform)

When I check the range of the dataset input images:
print("min:%f max:%f" %(train_set.data.min(), train_set.data.max()))
output result:min:0.000000 max:255.000000

I was expecting [0, 1] instead, I don't know why that is. Is there something wrong?

Output:
The reason why you have a range of [0,255] is that you are accessing the underlying data of the dataset via the data attribute. This means the transforms have not been applied yet to the data.
>>> train_transform = T.Compose([T.ToTensor()])
>>> train_set = torchvision.datasets.MNIST(
       root='.', train=True, download=True, transform=train_transform)

Your access of the data:
>>> f'min:{train_set.data.min()} max:{train_set.data.max()}'
min:0.000000 max:255.000000

You have to access the dataset by its proper interface in order for the transform pipeline to take effect. To make sure you could unroll the entire dataset's inputs into a tensor and look at its range:
>>> x, y = zip(*train_set)
>>> x_ = torch.stack(x)
>>> f'min:{x_.min()} max:{x_.max()}'
min:tensor(0.) max:tensor(1.)


Instruction: Why is my output predicting the same label using pretrained Alexnet in pytorch?

Input:
I'm attempting to use a pretrained alexnet model for CIFAR10 dataset however it always predicts everything as the same class. I use the exact same code except using alexnet untrained and it works as intended. Why is it doing this?
Here is my code:
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
net = models.alexnet(pretrained=True).to(device)

transform = transforms.Compose(
    [transforms.Resize((224, 224)),
     transforms.ToTensor(),
     transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,
                                          shuffle=True, num_workers=2, )

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,
                                         shuffle=True, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.005, momentum=0.9)

for epoch in range(3):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data
        inputs, labels = inputs.to(device),labels.to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')
            running_loss = 0.0

print('Finished Training')

After the code I print the accuracy of each class. And it predicts every image as a plane.

Output:
Pytorch AlexNet was trained on ImageNet so the classifier is with 1000 classes.
CIFAR10 is a 10 classes dataset.
You should create a new classifier before training with CIFAR10.
I found this post By Dr. Vaibhav Kumar that should explain how to do so.


Instruction: DataLoader returns multiple values sequentially instead of a list or tuple

Input:
def __init__():

def __len__():

def __getitem__(self, idx):
    cat_cols = (self.cat_cols.values.astype(np.float32))
    cont_cols = (self.cont_cols.values.astype(np.float32))
    label = (self.label.astype(np.int32))
    return (cont_cols[idx], cat_cols[idx], label[idx])

When I used the dataloader in the above class, I get the cont_cols, cat_cols and label as outputs with index 0, 1 and 2. Whereas I want them together. I have tried returning values as dictionary but then I have indexing issues.
I have to read the output of dataloader as
dl = DataLoader(dataset[0], batch_size = 1)

for i, data in enumerate(dl):
    if i == 0:
       cont = data
    if i == 1:
       cat = data
    if i == 2:
       label = data

Currently my output for
for i, data in enumerate(dl):
   print(i, data)

is
0 tensor([[3.2800e+02, 4.8000e+01, 1.0000e+03, 1.4069e+03, 4.6613e+05, 5.3300e+04,
0.0000e+00, 5.0000e+00, 1.0000e+00, 1.0000e+00, 2.0000e+00, 7.1610e+04,
6.5100e+03, 1.3020e+04, 5.2080e+04, 2.0040e+03]])
1 tensor([[ 2.,  1.,  1.,  4.,  2., 17.,  0.,  2.,  3.,  0.,  4.,  4.,  1.,  2.,
2., 10.,  1.]])
2 tensor([1], dtype=torch.int32)
What I want is the output to be accessed by data[0], data[1] and data[2] but the dataloader gives me back only data[0]. It is returning the cont_cols first, then cat_cols and then label.

Output:
I think you got confused here, your dataset can indeed return tuples but you have to handle it differently.
Your dataset is defined as:
class MyDataset(Dataset):
    def __init__(self):
        pass

    def __len__():
        pass

    def __getitem__(self, idx):
        cat_cols = (self.cat_cols.values.astype(np.float32))
        cont_cols = (self.cont_cols.values.astype(np.float32))
        label = (self.label.astype(np.int32))
        return (cont_cols[idx], cat_cols[idx], label[idx])

Then you define your dataset and data loader. Note, you should not provide dataset[0] here, but instead dataset:
>>> dataset = Dataset()
>>> dl = DataLoader(dataset, batch_size=1)

Then access your dataloader content in a loop:
>>> for cont, cat, label in dl:
...   print(cont, cat, label)


Instruction: Find PyTorch model parameters that don't contribute to loss

Input:
In PyTorch (v1.10) Distibuted DataParallel, unused parameters in a model that don't contribute to the final loss can raise a RuntimeError (as mentioned in this other question, this PyTorch forums thread).

"RuntimeError: Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your module has parameters that were not used in producing loss. You can enable unused parameter detection by passing the keyword argument find_unused_parameters=True to torch.nn.parallel.DistributedDataParallel, and by making sure all forward function outputs participate in calculating loss."

Although it's possible to inspect which parameters are affected at error-time (as mentioned above, or setting env var TORCH_DISTRIBUTED_DEBUG="INFO"), it seems like there should be a way to statically inspect a model to locate (and presumably prune or disable gradient on) parameters that aren't contributing to the current loss objective?
So given a torch.nn.Module-based model whose forward() function returns some loss tensor (maybe alongside others) - How can we programmatically, before starting to train, find all parameters (including nested modules) that aren't contributing to loss?

Output:
By default, PyTorch tensors that are the result of some computation record their history, that is their ancestors. This is needed for the backward pass to compute the gradient.
We can make use of this to find all tensors that contribute to some new tensors by just going through the whole history.
Note that this works for a static network that always has the same architecture. As soon as you have conditionals that e.g. depend on some intermediate value this won't work, and I claim in that case it is impossible to find what tensors are involved in advance. (It's similar to the halting problem.)
import torch
import torch.nn as nn
# Example of a simple network
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.x = nn.Parameter(torch.tensor([999999.0]))  # not contributing
        self.layers = nn.ModuleList([nn.Sequential(nn.Linear(1, 4), nn.Linear(4, 1)) for _ in range(3)])
    def forward(self, x):
        for m in self.layers: x = m(x) + x
        return x

net = Net()
x = torch.ones((1, 1))
# compute the forward pass to create the computation graph
y = net(x)

# use computation graph to find all contributing tensors
def get_contributing_params(y, top_level=True):
    nf = y.grad_fn.next_functions if top_level else y.next_functions
    for f, _ in nf:
        try:
            yield f.variable
        except AttributeError:
            pass  # node has no tensor
        if f is not None:
            yield from get_contributing_params(f, top_level=False)

contributing_parameters = set(get_contributing_params(y))
all_parameters = set(net.parameters())
non_contributing = all_parameters - contributing_parameters
print(non_contributing)  # returns the [999999.0] tensor


Instruction: Should I normalize or weight losses when combining them in pytorch?

Input:
network architecture
I have a neural network with 3 heads, one of them with a focal loss and two others with L1 losses. They are combined by summing:  loss = hm_loss + off_loss + wh_loss
However the range of typical values for loss elements are different. Is it an issue? Should I weight the loss elements, or should I normalize the network outputs?

Output:
This is a typical challenge when performing multi-task learning. There are many methods to handle this, but as for all things in this field, there is no single solution to solve them all. The most straightforward approach is to weigh the different loss components indeed. You can do so by performing a grid search or random search on the three weights or try and level the three components of your loss by looking at the orders of magnitude for each of them. The general idea behind this is if you're giving high precedence for one of the loss terms, then the gradient corresponding to this term will be much more prominent when performing back propagation and parameter update.
I recommend you read more on multi-task learning. For example you could start with Multi-Task Learning for Dense Prediction Tasks A Survey: Simon Vandenhende et al., in TPAMI'21.


Instruction: Efficiently insert alternate rows and columns in Python

Input:
I need to alternate Pytorch Tensors (similar to numpy arrays) with rows and columns of zeros. Like this:
Input => [[ 1,2,3],
           [ 4,5,6],
           [ 7,8,9]]

output => [[ 1,0,2,0,3],
           [ 0,0,0,0,0],
           [ 4,0,5,0,6],
           [ 0,0,0,0,0],
           [ 7,0,8,0,9]]

I am using the accepted answer in this question that proposes the following
def insert_zeros(a, N=1):
    # a : Input array
    # N : number of zeros to be inserted between consecutive rows and cols
    out = np.zeros( (N+1)*np.array(a.shape)-N,dtype=a.dtype)
    out[::N+1,::N+1] = a
    return out

The answers works perfectly, except that I need to perform this many times on many arrays and the time it takes has become the bottleneck. It is the step-sized slicing that takes most of the time.
For what it's worth, the matrices I am using it for are 4D, an example size of a matrix is 32x18x16x16 and I am inserting the alternate rows/cols only in the last two dimensions.
So my question is, is there another implementation with the same functionality but with reduced time?

Output:
I am not familiar to Pytorch, but to accelerate the code that you provided, I think JAX library will help a lot. So, if:
import numpy as np
import jax
import jax.numpy as jnp
from functools import partial

a = np.arange(10000).reshape(100, 100)
b = jnp.array(a)

@partial(jax.jit, static_argnums=1)
def new(a, N):
    out = jnp.zeros( (N+1)*np.array(a.shape)-N,dtype=a.dtype)
    out = out.at[::N+1,::N+1].set(a)
    return out

will improve the runtime about 10 times on GPU. It depends to array size and N (The increase in the sizes, the better performances). You can see Benchmarks on my Colab link based on the 4 answer proposed so far (JAX beats the others).
I believe that jax can be one of the best libraries for your case if you could adjust it on your problem (It is possible).


Instruction: Fastai Regression model with observation weight

Input:
Is it possible to have a costume mean squared error function with sample weight for each observation?
I am able to utilize the standard fastai training loop and I am able to implement this costume loss in PyTorch.
How to put that to fastai learner object on tabular data?
I know keras has this already implemented in the .fit method where sample_weight argument present.
def weighted_mse_loss(input, target, weight):
    return torch.sum(weight * (input - target) ** 2)

from fastai.tabular.all import *
import seaborn as sns

df = sns.load_dataset('tips')
df = df.assign(sample_weight = np.random.normal(size = df.shape[0], loc = 10, scale = 2))

y = ['total_bill']
cont = ['tip']
cat = ['sex', 'smoker', 'day', 'time', 'size']

procs = [Normalize, Categorify]

df["Y"] = np.log(df[y] + 1)

MIN = df["Y"].min()
MAX = df["Y"].max()

splits =  RandomSplitter(valid_pct=0.2)(range_of(df))

to = TabularPandas(
    df,
    procs=procs,
    cat_names=cat,
    cont_names=cont,
    y_names="Y",
    splits=splits,
    y_block=RegressionBlock(n_out = 1),
)

dls = to.dataloaders(
    bs=64, shuffle_train=True
)

config = tabular_config(
        embed_p=0.05,
        y_range=[0, MAX * 1.1],
        bn_final=False,
        ps=[0.05, 0.05, 0.05],
    )

learner = tabular_learner(
        dls,
        layers=[1000, 500, 250],
        config=config,
        wd=0.2,
        metrics=[rmse,],
    )

learner.fit_one_cycle(40, lr_max = 0.01,
                          wd = 0.1)

Output:
I am using this workaround:

in y_names for TabularPandas, you can return the tuple of (weight, y) as
to = TabularPandas(df,
                   procs=procs,
                   cat_names=cat,
                   cont_names=cont,
                   y_names=["sample_weight","Y"],
                   splits=splits,
                   y_block=RegressionBlock(n_out = 1))

In your loss function, split your target to (weights, target) and apply weights to loss, e.g.:
 class SampleWeightedCE(torch.nn.modules.loss._Loss):
     def __init__(self):
         super(SampleWeightedCE, self).__init__()
         self.ce_loss = torch.nn.BCEWithLogitsLoss(reduction='none')

 def forward(self, output, tgt):
     weights = tgt[:,0].unsqueeze(1)
     target = tgt[:,1].unsqueeze(1)

     losses = self.ce_loss(output, target) * weights
     return torch.sum(losses) / torch.sum(weights)

If you want to measure metrics, you can use the same workaround, such as:
def accuracy_W(inp, tgt, thresh=0.5, sigmoid=True):
 weights = tgt[:,0].unsqueeze(1)
 target = tgt[:,1].unsqueeze(1)

 if sigmoid: inp = inp.sigmoid()
 classes = (inp >= thresh)
 m_target = (target >= 0.5)
 correct = (m_target == classes)
 return torch.sum(weights * correct) / torch.sum(weights)

In get_preds() or predict(), you need to split the target
 y_prob, y_out = learn.get_preds(ds_idx=1, with_input=False, with_loss=False, reorder=False)
 weights = y_out[:,0]
 target = y_out[:,1]


Instruction: pytorch tensor change dimensionallity to count adjacent values

Input:
My objective it to count all adjacent unique values of a tensor x.
Say my tensor is (x looks like a list but it is a pytorch tensor)
x = [1,2,1,2,4,5]

I would want my output to be:
[1,2] = 2
[2,1] = 1
[2,4] = 1
[4,5] = 1

I thought about changing the dimensionality of the tensor to look like:
x = [[1,2],[2,1],[1,2],[2,4],[4,5]]

using tensor.view but couldn't find a solution that works for a tensor of any length.
Any ideas if this is even the best way to go about this? is there some built-in function?

Output:
As @ihdv showed, you can stack shifted views of x with torch.stack or torch.vstack in order to get a tensor of pairs with overlapping windows.
>>> p = torch.vstack((x[:-1], x[1:]))
tensor([[1., 2., 1., 2., 4.],
        [2., 1., 2., 4., 5.]])

Then you can apply torch.unique on it to get the statistics:
>>> p.unique(dim=1, return_counts=True)
(tensor([[1., 2., 2., 4.],
         [2., 1., 4., 5.]]), tensor([2, 1, 1, 1]))


Instruction: How to get the input and output feature maps of a CNN model?

Input: I am trying to find the dimensions of an image as it goes through a convolutional neural network at each layer. So for instance, if there is max-pooling or convolution being applied, I’d like to know the shape of the image at that layer, for all layers. I know I can use the nOut=image+2p-f / s + 1 formula but it would be too tedious and complex given the size of the PyTorch model. Is there a simple way to do this, perhaps a visualization tool/script or something?

Output:
You can use the torchinfo library: https://github.com/TylerYep/torchinfo
Let's take their example:
from torchinfo import summary

model = ConvNet()
batch_size = 16
summary(model, input_size=(batch_size, 1, 28, 28))

Here (1, 28, 28) is the input's size, which is (Channel, Width, Height) of the image respectively.
The library will print:
================================================================================================================
Layer (type:depth-idx)          Input Shape          Output Shape         Param #            Mult-Adds
================================================================================================================
SingleInputNet                  --                   --                   --                  --
├─Conv2d: 1-1                   [7, 1, 28, 28]       [7, 10, 24, 24]      260                1,048,320
├─Conv2d: 1-2                   [7, 10, 12, 12]      [7, 20, 8, 8]        5,020              2,248,960
├─Dropout2d: 1-3                [7, 20, 8, 8]        [7, 20, 8, 8]        --                 --
├─Linear: 1-4                   [7, 320]             [7, 50]              16,050             112,350
├─Linear: 1-5                   [7, 50]              [7, 10]              510                3,570
================================================================================================================
Total params: 21,840
Trainable params: 21,840
Non-trainable params: 0
Total mult-adds (M): 3.41
================================================================================================================
Input size (MB): 0.02
Forward/backward pass size (MB): 0.40
Params size (MB): 0.09
Estimated Total Size (MB): 0.51
================================================================================================================

I think 7 is wrong in this output though. It should be 16.


Instruction: Matrix 2D on Convolutional Netowrk

Input:
that may be a silly question, but I wanted to use a convolutional neural network in my deep reinforcement learning project and I got a problem I don't understand.
In my project I want to insert into network matrix 6x7 which should be equivalent to black and white picture of 6x7 size (42 pixels) right?
class CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.model = torch.nn.Sequential()
        self.model.add_module("conv_1", torch.nn.Conv2d(in_channels=1, out_channels=16, kernel_size=4, stride = 1))
        self.model.add_module("relu_1", torch.nn.ReLU())
        self.model.add_module("max_pool", torch.nn.MaxPool2d(2))
        self.model.add_module("conv_2", torch.nn.Conv2d(in_channels=16, out_channels=16, kernel_size=4, stride = 1))
        self.model.add_module("relu_2", torch.nn.ReLU())
        self.model.add_module("flatten", torch.nn.Flatten())

        self.model.add_module("linear", torch.nn.Linear(in_features=16*16*16, out_features=7))

    def forward(self, x):
        x = self.model(x)
        return x

In conv1 in_channels=1 because I have got only 1 matrix (if it was image recognition that means 1 color). Other in_channels and out_channels are kind of random until linear.  I have no idea where I should insert the size of a matrix, but the final output should be a size of 7 which i typed in linear.
The error i get is:
RuntimeError: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [6, 7]

Output:
There are a few problems with your code. First, the reason you're getting that error message is because the CNN is expecting a tensor with shape (N, Cin, Hin, Win), where:

N is the batch size
Cin is the number of input channels
Hin is the input image pixel height
Win is the input image pixel width

You're only providing the width and height dimensions. You need to explicitly add a channels and batch dimension, even if the value of those dimensions is only 1:
model = CNN()

example_input = torch.randn(size=(6, 7)) # this is your input image

print(example_input.shape) # should be (6, 7)

output = model(example_input) # you original error

example_input = example_input.unsqueeze(0).unsqueeze(0) # adds batch and channels dimension

print(example_input.shape) # should now be (1, 1, 6, 7)

output = model(example_input) # no more error!

You'll note however, you get a different error now:
RuntimeError: Calculated padded input size per channel: (1 x 2). Kernel size: (4 x 4). Kernel size can't be greater than actual input size

This is because after the first conv layer, your data is of shape 1x2, but your kernel size for the second layer is 4, which makes the operation impossible. An input image of size 6x7 is quite small, either reduce the kernel size to something that works, or use a larger images.
Here's a working example:
import torch
from torch import nn

class CNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.model = torch.nn.Sequential()
        self.model.add_module(
            "conv_1",
            torch.nn.Conv2d(in_channels=1, out_channels=16, kernel_size=2, stride=1),
        )
        self.model.add_module("relu_1", torch.nn.ReLU())
        self.model.add_module("max_pool", torch.nn.MaxPool2d(2))
        self.model.add_module(
            "conv_2",
            torch.nn.Conv2d(in_channels=16, out_channels=16, kernel_size=2, stride=1),
        )
        self.model.add_module("relu_2", torch.nn.ReLU())
        self.model.add_module("flatten", torch.nn.Flatten())

        self.model.add_module("linear", torch.nn.Linear(in_features=32, out_features=7))

    def forward(self, x):
        x = self.model(x)
        return x

model = CNN()
x = torch.randn(size=(6, 7))
x = x.unsqueeze(0).unsqueeze(0)
output = model(x)
print(output.shape) # has shape (1, 7)

Note, I changed the kernel_size to 2, and the final linear layer has an input size of 32. Also, the output has shape (1, 7), the 1 is the batch_size, which in our case was only 1. If you want just the 7 output features, just use x = torch.squeeze(x).