Instruction: Issue when Re-implement Matrix Factorization in Pytorch

Input:
I try to implement matrix factorization in Pytorch as the data extractor and model.
The original model is written in mxnet. Here I try to use the same idea in Pytorch.
Here is my code, it can be runned directly in codelab
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader

import collections
from collections import defaultdict
from IPython import display
import math
from matplotlib import pyplot as plt
import os
import pandas as pd
import random
import re
import shutil
import sys
import tarfile
import time
import requests
import zipfile
import hashlib

# ============data obtained, not change the original code
DATA_HUB= {}

# Defined in file: ./chapter_multilayer-perceptrons/kaggle-house-price.md
def download(name, cache_dir=os.path.join('..', 'data')):
    """Download a file inserted into DATA_HUB, return the local filename."""
    assert name in DATA_HUB, f"{name} does not exist in {DATA_HUB}."
    url, sha1_hash = DATA_HUB[name]
    os.makedirs(cache_dir, exist_ok=True)
    fname = os.path.join(cache_dir, url.split('/')[-1])
    if os.path.exists(fname):
        sha1 = hashlib.sha1()
        with open(fname, 'rb') as f:
            while True:
                data = f.read(1048576)
                if not data:
                    break
                sha1.update(data)
        if sha1.hexdigest() == sha1_hash:
            return fname  # Hit cache
    print(f'Downloading {fname} from {url}...')
    r = requests.get(url, stream=True, verify=True)
    with open(fname, 'wb') as f:
        f.write(r.content)
    return fname

# Defined in file: ./chapter_multilayer-perceptrons/kaggle-house-price.md
def download_extract(name, folder=None):
    """Download and extract a zip/tar file."""
    fname = download(name)
    base_dir = os.path.dirname(fname)
    data_dir, ext = os.path.splitext(fname)
    if ext == '.zip':
        fp = zipfile.ZipFile(fname, 'r')
    elif ext in ('.tar', '.gz'):
        fp = tarfile.open(fname, 'r')
    else:
        assert False, 'Only zip/tar files can be extracted.'
    fp.extractall(base_dir)
    return os.path.join(base_dir, folder) if folder else data_dir

#1. obtain dataset
DATA_HUB['ml-100k'] = ('http://files.grouplens.org/datasets/movielens/ml-100k.zip',
    'cd4dcac4241c8a4ad7badc7ca635da8a69dddb83')

def read_data_ml100k():
    data_dir = download_extract('ml-100k')
    names = ['user_id', 'item_id', 'rating', 'timestamp']
    data = pd.read_csv(os.path.join(data_dir, 'u.data'), '\t', names=names,
                       engine='python')
    num_users = data.user_id.unique().shape[0]
    num_items = data.item_id.unique().shape[0]
    return data, num_users, num_items

# 2. Split data
#@save
def split_data_ml100k(data, num_users, num_items,
                      split_mode='random', test_ratio=0.1):
    """Split the dataset in random mode or seq-aware mode."""
    if split_mode == 'seq-aware':
        train_items, test_items, train_list = {}, {}, []
        for line in data.itertuples():
            u, i, rating, time = line[1], line[2], line[3], line[4]
            train_items.setdefault(u, []).append((u, i, rating, time))
            if u not in test_items or test_items[u][-1] < time:
                test_items[u] = (i, rating, time)
        for u in range(1, num_users + 1):
            train_list.extend(sorted(train_items[u], key=lambda k: k[3]))
        test_data = [(key, *value) for key, value in test_items.items()]
        train_data = [item for item in train_list if item not in test_data]
        train_data = pd.DataFrame(train_data)
        test_data = pd.DataFrame(test_data)
    else:
        mask = [True if x == 1 else False for x in np.random.uniform(
            0, 1, (len(data))) < 1 - test_ratio]
        neg_mask = [not x for x in mask]
        train_data, test_data = data[mask], data[neg_mask]
    return train_data, test_data

#@save
def load_data_ml100k(data, num_users, num_items, feedback='explicit'):
    users, items, scores = [], [], []
    inter = np.zeros((num_items, num_users)) if feedback == 'explicit' else {}
    for line in data.itertuples():
        user_index, item_index = int(line[1] - 1), int(line[2] - 1)
        score = int(line[3]) if feedback == 'explicit' else 1
        users.append(user_index)
        items.append(item_index)
        scores.append(score)
        if feedback == 'implicit':
            inter.setdefault(user_index, []).append(item_index)
        else:
            inter[item_index, user_index] = score
    return users, items, scores, inter

#@save
def split_and_load_ml100k(split_mode='seq-aware', feedback='explicit',
                          test_ratio=0.1, batch_size=256):
    data, num_users, num_items = read_data_ml100k()
    train_data, test_data = split_data_ml100k(data, num_users, num_items, split_mode, test_ratio)
    train_u, train_i, train_r, _ = load_data_ml100k(train_data, num_users, num_items, feedback)
    test_u, test_i, test_r, _ = load_data_ml100k(test_data, num_users, num_items, feedback)

    # Create Dataset
    train_set = MyData(np.array(train_u), np.array(train_i), np.array(train_r))
    test_set = MyData(np.array(test_u), np.array(test_i), np.array(test_r))

    # Create Dataloader
    train_iter = DataLoader(train_set, shuffle=True, batch_size=batch_size)
    test_iter = DataLoader(test_set, batch_size=batch_size)

    return num_users, num_items, train_iter, test_iter

class MyData(Dataset):
  def __init__(self, user, item, score):
    self.user = torch.tensor(user)
    self.item = torch.tensor(item)
    self.score = torch.tensor(score)

  def __len__(self):
    return len(self.user)

  def __getitem__(self, idx):
    return self.user[idx], self.item[idx], self.score[idx]

# create a nn class (just-for-fun choice :-)
class RMSELoss(nn.Module):
    def __init__(self, eps=1e-6):
        '''You should be careful with NaN which will appear if the mse=0, adding self.eps'''
        super().__init__()
        self.mse = nn.MSELoss()
        self.eps = eps

    def forward(self,yhat,y):
        loss = torch.sqrt(self.mse(yhat,y) + self.eps)
        return loss

class MF(nn.Module):
    def __init__(self, num_factors, num_users, num_items, **kwargs):
        super(MF, self).__init__(**kwargs)
        self.P = nn.Embedding(num_embeddings=num_users, embedding_dim=num_factors)
        self.Q = nn.Embedding(num_embeddings=num_items, embedding_dim=num_factors)
        self.user_bias = nn.Embedding(num_users, 1)
        self.item_bias = nn.Embedding(num_items, 1)

    def forward(self, user_id, item_id):
        P_u = self.P(user_id)
        Q_i = self.Q(item_id)

        b_u = self.user_bias(user_id)
        b_i = self.item_bias(item_id)

        outputs = (P_u * Q_i).sum() + b_u.squeeze() + b_i.squeeze()
        return outputs

# train
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Hyper parameters
num_epochs = 50
batch_size = 512
lr = 0.001

num_users, num_items, train_iter, test_iter = split_and_load_ml100k(test_ratio=0.1, batch_size=batch_size)

model = MF(30, num_users, num_items).to(device)

# Loss and Optimizer
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
criterion = RMSELoss()

# Train the Model
train_rmse = []
test_rmse = []
for epoch in range(num_epochs):
    train_loss = 0
    num_train = 0
    model.train()
    for users, items, scores in train_iter:
        users = users.to(device)
        items = items.to(device)
        scores = scores.float().to(device)

        # Forward pass
        outputs = model(users, items)
        loss = criterion(outputs, scores)

        # Backward and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        train_loss += loss.item()
        num_train += scores.shape[0]

    train_rmse.append(train_loss / num_train)

    model.eval()
    test_loss = 0
    num_test = 0
    with torch.no_grad():
        for users, items, scores in test_iter:
            users = users.to(device)
            items = items.to(device)
            scores = scores.float().to(device)

            outputs = model(users, items)
            loss = criterion(outputs, scores)

            test_loss += loss.item()
            num_test += scores.shape[0]

    test_rmse.append(test_loss / num_test)

# plot
%matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')

x = list(range(num_epochs))
fig = plt.figure()
ax = plt.axes()

plt.plot(x, train_rmse, label='train_rmse');
plt.plot(x, test_rmse, label='test_rmse');

leg = ax.legend();

I got the result

The MXNET result is here

Why I cannot get a beautiful shape. And my train_rmse is larger than test_rmse.

Output:
I modified your code a bit and got a similar result with mxnet's. Here is the code in colab.

model. you missed axis=1 in the summation operation.

outputs = (P_u * Q_i).sum(axis=1) + b_u.squeeze() + b_i.squeeze()

The default sum operation will sum all the elements in a tensor and produces a scalar. It is fine to add a scalar to a tensor so you didn't catch an error.

optimizer. I used the same optimizer - Adam as the mxnet's implementation. Similarly, I also added weight decay.

optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd)

initialization. Initialize the weights with normal distribution.

nn.init.normal_(self.P.weight, std=0.01)
nn.init.normal_(self.Q.weight, std=0.01)
nn.init.normal_(self.user_bias.weight, std=0.01)
nn.init.normal_(self.item_bias.weight, std=0.01)

Other,
You don't need to add num_train with the batch size. The loss has already been divided by the batch size in the MSELoss.
num_train += 1


Instruction: How to sort a 3d tensor by coordinates in last dimension (pytorch)

Input:
I have a tensor with shape [bn, k, 2]. The last dimension are coordinates and I want each batch to be sorted independently depending on the y coordinate ([:, :, 0]). My approach looks something like this:
import torch
a = torch.randn(2, 5, 2)
indices = a[:, :, 0].sort()[1]
a_sorted = a[:, indices]

print(a)
print(a_sorted)

So far so good, but I now it sorts both batches according to both index lists, so I get 4 batches in total:
a
tensor([[[ 0.5160,  0.3257],
         [-1.2410, -0.8361],
         [ 1.3826, -1.1308],
         [ 0.0338,  0.1665],
         [-0.9375, -0.3081]],

        [[ 0.4140, -1.0962],
         [ 0.9847, -0.7231],
         [-0.0110,  0.6437],
         [-0.4914,  0.2473],
         [-0.0938, -0.0722]]])

a_sorted
tensor([[[[-1.2410, -0.8361],
          [-0.9375, -0.3081],
          [ 0.0338,  0.1665],
          [ 0.5160,  0.3257],
          [ 1.3826, -1.1308]],

         [[ 0.0338,  0.1665],
          [-0.9375, -0.3081],
          [ 1.3826, -1.1308],
          [ 0.5160,  0.3257],
          [-1.2410, -0.8361]]],

        [[[ 0.9847, -0.7231],
          [-0.0938, -0.0722],
          [-0.4914,  0.2473],
          [ 0.4140, -1.0962],
          [-0.0110,  0.6437]],

         [[-0.4914,  0.2473],
          [-0.0938, -0.0722],
          [-0.0110,  0.6437],
          [ 0.4140, -1.0962],
          [ 0.9847, -0.7231]]]])

As you can see, I want only the 1st and the 4th batch to be returned. How do I do that?

Output:
What you want: concatenation of a[0, indices[0]] and a[1, indices[1]].
What you coded: concatenation of a[0, indices] and a[1, indices].
The issue you are facing is because the indices returned by sort are shaped like the first dimensions, but the values are only indices into the second dimension. When you go to use these, you want to match indices[0] on a[0], but pytorch doesn't do this implicitly (because fancy indexing is very powerful, and needs this syntax for it's power). So, all you have to do is give a parallel list of indices for the first dimension.
i.e. You want to use something like: a[[[0], [1]], indices].
To generalise this a bit more, you may use something like:
n = a.shape[0]
first_indices = torch.arange(n)[:, None]
a[first_indices, indices]

This is a little tricksy, so here's an example:
>>> a = torch.randn(2,4,2)
>>> a
tensor([[[-0.2050, -0.1651],
         [ 0.5688,  1.0082],
         [-1.5964, -0.9236],
         [ 0.3093, -0.2445]],

        [[ 1.0586,  1.0048],
         [ 0.0893,  2.4522],
         [ 2.1433, -1.2428],
         [ 0.1591,  2.4945]]])
>>> indices = a[:, :, 0].sort()[1]
>>> indices
tensor([[2, 0, 3, 1],
        [1, 3, 0, 2]])
>>> a[:, indices]
tensor([[[[-1.5964, -0.9236],
          [-0.2050, -0.1651],
          [ 0.3093, -0.2445],
          [ 0.5688,  1.0082]],

         [[ 0.5688,  1.0082],
          [ 0.3093, -0.2445],
          [-0.2050, -0.1651],
          [-1.5964, -0.9236]]],

        [[[ 2.1433, -1.2428],
          [ 1.0586,  1.0048],
          [ 0.1591,  2.4945],
          [ 0.0893,  2.4522]],

         [[ 0.0893,  2.4522],
          [ 0.1591,  2.4945],
          [ 1.0586,  1.0048],
          [ 2.1433, -1.2428]]]])
>>> a[0, indices]
tensor([[[-1.5964, -0.9236],
         [-0.2050, -0.1651],
         [ 0.3093, -0.2445],
         [ 0.5688,  1.0082]],

        [[ 0.5688,  1.0082],
         [ 0.3093, -0.2445],
         [-0.2050, -0.1651],
         [-1.5964, -0.9236]]])
>>> a[1, indices]
tensor([[[ 2.1433, -1.2428],
         [ 1.0586,  1.0048],
         [ 0.1591,  2.4945],
         [ 0.0893,  2.4522]],

        [[ 0.0893,  2.4522],
         [ 0.1591,  2.4945],
         [ 1.0586,  1.0048],
         [ 2.1433, -1.2428]]])
>>> a[0, indices[0]]
tensor([[-1.5964, -0.9236],
        [-0.2050, -0.1651],
        [ 0.3093, -0.2445],
        [ 0.5688,  1.0082]])
>>> a[1, indices[1]]
tensor([[ 0.0893,  2.4522],
        [ 0.1591,  2.4945],
        [ 1.0586,  1.0048],
        [ 2.1433, -1.2428]])
>>> a[[[0], [1]], indices]
tensor([[[-1.5964, -0.9236],
         [-0.2050, -0.1651],
         [ 0.3093, -0.2445],
         [ 0.5688,  1.0082]],

        [[ 0.0893,  2.4522],
         [ 0.1591,  2.4945],
         [ 1.0586,  1.0048],
         [ 2.1433, -1.2428]]])


Instruction: BERT DataLoader: Difference between shuffle=True vs Sampler?

Input:
I trained a DistilBERT model with DistilBertForTokenClassification on ConLL data fro predicting NER. Training seem to have completed with no problems but I have 2 problems during evaluation phase.

I'm getting negative loss value

During training, I used shuffle=True for DataLoader. But during evaluation, when I do shuffle=True for DataLoader, I get very poor metric results(f_1, accuracy, recall etc). But if I do shuffle = False or use a Sampler instead of shuffling I get pretty good metric results. I'm wondering if there is anything wrong with my code.

Here is the evaluation code:

print('Prediction started on test data')
model.eval()

eval_loss = 0
predictions , true_labels = [], []

for batch in val_loader:
  b_input_ids = batch['input_ids'].to(device)
  b_input_mask = batch['attention_mask'].to(device)
  b_labels = batch['labels'].to(device)

  with torch.no_grad():
      outputs = model(b_input_ids,
                      attention_mask=b_input_mask)

  logits = outputs[0]
  logits = logits.detach().cpu().numpy()
  label_ids = b_labels.detach().cpu().numpy()

  predictions.append(logits)
  true_labels.append(label_ids)

  eval_loss += outputs[0].mean().item()

print('Prediction completed')
eval_loss = eval_loss / len(val_loader)
print("Validation loss: {}".format(eval_loss))

out:
Prediction started on test data
Prediction completed
Validation loss: -0.2584906197858579

I believe I'm calculating the loss wrong here. Is it possible to get negative loss values with BERT?
For DataLoader, if I use the code snippet below, I have no problems with the metric results.
val_sampler = SequentialSampler(val_dataset)
val_loader = DataLoader(val_dataset, sampler=val_sampler, batch_size=128)

Bu if I do this one I get very poor metric results
val_loader = DataLoader(val_dataset, batch_size=128, shuffle=True)

Is it normal that I'm getting vastly different results with shuffle=True vs shuffle=False ?
code for the metric calculation:
metric = load_metric("seqeval")
results = metric.compute(predictions=true_predictions, references=true_labels)
results

out:
{'LOCATION': {'f1': 0.9588207767898924,
  'number': 2134,
  'precision': 0.9574766355140187,
  'recall': 0.9601686972820993},
 'MISC': {'f1': 0.8658965344048217,
  'number': 995,
  'precision': 0.8654618473895582,
  'recall': 0.8663316582914573},
 'ORGANIZATION': {'f1': 0.9066332916145182,
  'number': 1971,
  'precision': 0.8947628458498024,
  'recall': 0.9188229325215627},
 'PERSON': {'f1': 0.9632426988922457,
  'number': 2015,
  'precision': 0.9775166070516096,
  'recall': 0.9493796526054591},
 'overall_accuracy': 0.988255561629313,
 'overall_f1': 0.9324058459808882,
 'overall_precision': 0.9322748349023465,
 'overall_recall': 0.932536893886156}

The above metrics are printed when I use Sampler or shuffle=False. If I use shuffle=True, I get:
{'LOCATION': {'f1': 0.03902284263959391,
  'number': 2134,
  'precision': 0.029496402877697843,
  'recall': 0.057638238050609185},
 'MISC': {'f1': 0.010318142734307824,
  'number': 995,
  'precision': 0.009015777610818933,
  'recall': 0.012060301507537688},
 'ORGANIZATION': {'f1': 0.027420984269014285,
  'number': 1971,
  'precision': 0.019160951996772892,
  'recall': 0.04819888381532217},
 'PERSON': {'f1': 0.02119907254057635,
  'number': 2015,
  'precision': 0.01590852597564007,
  'recall': 0.03176178660049628},
 'overall_accuracy': 0.5651741788003777,
 'overall_f1': 0.02722600361161272,
 'overall_precision': 0.020301063389034663,
 'overall_recall': 0.041321152494729445}

UPDATE: I modified loss code for evaluation. There seems to be no problem with this code. You can see the new code below:
print('Prediction started on test data')
model.eval()

eval_loss = 0
predictions , true_labels = [], []

for batch in val_loader:

  b_labels = batch['labels'].to(device)

  batch = {k:v.type(torch.long).to(device) for k,v in batch.items()}

  with torch.no_grad():
      outputs = model(**batch)

      loss, logits = outputs[0:2]
      logits = logits.detach().cpu().numpy()
      label_ids = b_labels.detach().cpu().numpy()

      predictions.append(logits)
      true_labels.append(label_ids)

      eval_loss += loss

print('Prediction completed')
eval_loss = eval_loss / len(val_loader)
print("Validation loss: {}".format(eval_loss))

Though I still haven't got an asnwer to the DataLoader question.
Also I jsut realised when I do print(model.eval()) I still get dropouts from the model in evaluation mode.

Output:
As far as I understand, the answer is pretty simple:
"I saw my father do it this way, and his father was also doing it this way, so I'm also doing it this way".
I've looked around a lot of notebooks to see how people were loading the data for validation and in every notebook I saw that people were using the Sequential Sampler for validation. Nobody uses Shuffling or Random Sampling during validation. I don't exactly know why, but this is the case. So if anyone visiting this post was wondering the same thing, the answer is basically what I quoted above.
Also, I edited the original post for the loss problem I was having. I was calculating it wrong. Apperantly Bert reutrns loss at index 0 of the output (outputs[0]) if you also feed the model the original labels. In the first code snippet, when I was getting the outputs from the model, I was not feeding the model with the original labels, so it was not returning the loss value at index 0, but returning only the logits.
Basically what you need to do is:
outputs = model(input_ids, mask, label=label)
loss = outputs[0]
logits = outputs[1]


Instruction: PyTorch: Shuffle DataLoader

Input:
There are several scenarios that make me confused about shuffling the data loader, which are as follows.
I set the “shuffle” parameter to False on both train_loader and valid_loader. then the results I get are as follows
Epoch 1/4    loss=0.8802     val_loss=0.8202     train_acc=0.63      val_acc=0.63
Epoch 2/4    loss=0.6993     val_loss=0.6500     train_acc=0.66      val_acc=0.72
Epoch 3/4    loss=0.5363     val_loss=0.5385     train_acc=0.76      val_acc=0.80
Epoch 4/4    loss=0.4055     val_loss=0.5130     train_acc=0.85      val_acc=0.81

I set the “shuffle” parameter to True on train_loader and False to valid_loader. then the results I get are as follows
Epoch 1/4    loss=0.8928     val_loss=0.8284     train_acc=0.63      val_acc=0.63
Epoch 2/4    loss=0.7308     val_loss=0.6263     train_acc=0.61      val_acc=0.73
Epoch 3/4    loss=0.5594     val_loss=0.5046     train_acc=0.54      val_acc=0.81
Epoch 4/4    loss=0.4304     val_loss=0.4525     train_acc=0.49      val_acc=0.82

Based on that result, my training accuracy has a worse performance when I shuffle train_loader.
And this is a snippet of my code.
for epoch in range(n_epochs):
    model.train()
    avg_loss = 0.
    train_preds = np.zeros((len(train_X),len(le.classes_)))

    for i, (x_batch, y_batch) in enumerate(train_loader):
        y_pred = model(x_batch)
        loss = loss_fn(y_pred, y_batch)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        avg_loss += loss.item() / len(train_loader)
        train_preds[i * batch_size:(i+1) * batch_size] = F.softmax(y_pred).cpu().detach().numpy()

    train_accuracy = sum(train_preds.argmax(axis=1) == y_train)/len(y_train)

    model.eval()
    avg_val_loss = 0.
    val_preds = np.zeros((len(x_cv),len(le.classes_)))

    for i, (x_batch, y_batch) in enumerate(valid_loader):
        y_pred = model(x_batch).detach()
        avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)
        val_preds[i * batch_size:(i+1) * batch_size] =F.softmax(y_pred).cpu().numpy()
    val_accuracy = sum(val_preds.argmax(axis=1)==y_test)/len(y_test)

Did I make a mistake calculating the training accuracy? Thanks in advance

Output:
You are comparing the shuffled predictions with the un-shuffled labels. To fix that, count the number of accurate predictions in every iteration, and compute the overall accuracy at the end.
for epoch in range(n_epochs):
    model.train()
    avg_loss = 0.
    total_correct = 0
    total_samples = 0

    for i, (x_batch, y_batch) in enumerate(train_loader):
        y_pred = model(x_batch)
        loss = loss_fn(y_pred, y_batch)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        avg_loss += loss.item() / len(train_loader)
        total_correct += (torch.argmax(y_pred, 1) == y_batch).sum()
        total_samples += y_batch.shape[0]

    train_accuracy = total_correct / total_samples

(I haven't tested this code)


Instruction: Is there a better way to calculate loss for multi-task DNN modeling?

Input:
Suppose there are over one thousand tasks in the multi-task deep learning. More than a thousand columns of labels. Each task (column) has a specific weight in this case. It would take such long time to loop over each task to calculate the sum of loss using the following code snippet.
criterion = nn.MSELoss()

outputs = model(inputs)
loss = torch.tensor(0.0).to(device)
for j, w in enumerate(weights):
    # mask keeping labeled molecules for each task
    mask = labels[:, j] >= 0.0
    if len(labels[:, j][mask]) != 0:
        # the loss is the sum of each task/target loss.
        # there are labeled samples for this task, so we add it's loss
        loss += criterion(outputs[j][mask], labels[:, j][mask].view(-1, 1)) * w

This dataset was quite small. The dataset has 10K rows and 1024 columns and the labels are a 10K * 160 sparse matrix. Each of those 160 columns is one task. Batch size is 32.  Below are the shapes of outputs, labels, weights:
len(outputs[0]), len(outputs)
(32, 160)
weights.shape
torch.Size([160])
labels.shape
torch.Size([32, 160])

But what I really want to try is one dataset which has over 1M rows and 1024 features and over 10K labels. The labels are sparse of course.
**update**
Thanks for you suggestions and code, Shai. I modified the code a little bit as follows, but the loss was the same as your code.

all_out = torch.cat(outputs).view(len(outputs), -1).T
all_mask = labels != -100.0
err = (all_out - labels) ** 2  # raw L2
err = all_mask * err  # mask only the relevant entries in the err
mask_nums = all_mask.sum(axis=0)
err = err * weights[None, :]  # weight each task
err = err / mask_nums[None, :]
err[err != err] = torch.tensor([0.0], requires_grad=True).to(device) # replace nan to 0.0
loss = err.sum()

A newly raised question is the loss can't get back propagated. Only the loss of the first batch was calculated. The following batches got a loss of 0.0.

Epoch: [1/20], Step: [1/316], Loss: 4.702103614807129
Epoch: [1/20], Step: [2/316], Loss: 0.0
Epoch: [1/20], Step: [3/316], Loss: 0.0
Epoch: [1/20], Step: [4/316], Loss: 0.0
Epoch: [1/20], Step: [5/316], Loss: 0.0
The loss was 0 and outputs was 32* 160 of nan after the first batch.

Output:
How is your loss different than:
all_out = torch.cat([o_[:, None] for o_ in outputs], dim=1)  # all_out has shape 32x160
all_mask = labels >= 0

err = (all_out - labels) ** 2  # raw L2
err = all_mask * err  # mask only the relevant entries in the err
err = err * weights[None, :]  # weight each task
err = err.sum()

There might be a slight issue here with the summation - you might need to weight by the number of 1s in each column of all_mask.


Instruction: What does calling self within a class do?

Input:
I noticed in the documentation for Pytorch Lightning, it was mentioned you can call the forward method from another method in the same class just by calling self(x). I haven't been able to find any info about how this works. I always thought you would call the method using self.forward
Evidently, it calls the forward method but how? Is there any python documentation about what's going on?
I found this at the following URL:
https://pytorch-lightning.readthedocs.io/en/stable/new-project.html
The specific code fragment is this:
def training_step(self, batch, batch_idx):
    ...
    z = self(x)

Output:
Generally speaking, in python, when "calling" an object, you are invoking its __call__ method. That is,
  self(x)

is equivalent to
  self.__call__(x)

For pytorch nn.Module (and all derivative classes) __call__ wraps around the module's forward function, therefore, from your perspective self(x) is basically forwarding x through the module self.


Instruction: Unflatten a tensor back to an image

Input:
I am working on GANs and I want to visualize the image formed.
For this, I was trying
def show_images(image_tensor, num_images=9, size=(1, 28, 28)):
    image_unflat = image_tensor.detach().cpu.view(-1, *size)
    image_grid = make_grid(image_unflat[:num_images], nrow=3)
    plt.imshow(image_grid.permute(1, 2, 0).squeeze())
    plt.show()

but when I am trying to show_image(some_tensor), I am getting an error as
image_unflat = image_tensor.detach().cpu.view(-1, *size)
AttributeError: 'builtin_function_or_method' object has no attribute 'view'

Here, the size of some_tensor is N x 784.

Output:
You need to call cpu() before broadcasting with view.
image_unflat = image_tensor.detach().cpu().view(-1, *size)


Instruction: How to correctly use Cross Entropy Loss vs Softmax for classification?

Input:
I want to train a multi class classifier using Pytorch.
Following the official Pytorch doc shows how to use a nn.CrossEntropyLoss() after a last layer of type nn.Linear(84, 10).
However, I remember this is what Softmax does.
This leaves me confused.

How to train a "standard" classification network in the best way?
If the network has a final linear layer, how to infer the probabilities per class?
If the network has a final softmax layer, how to train the network (which loss, and how)?

I found this thread on the Pytorch forum, which likely answers all that, but I couldn't compile it into working and readable Pytorch code.

My assumed answers:

Like the doc says.
Exponentiation of the outputs of the linear layer, which are really logits (log probalbilities).
I don't understand.

Output:
I think that it's important to understand softmax and cross-entropy, at least from a practical point of view. Once you have a grasp on these two concepts then it should be clear how they may be "correctly" used in the context of ML.
Cross Entropy H(p, q)
Cross-entropy is a function that compares two probability distributions. From a practical standpoint it's probably not worth getting into the formal motivation of cross-entropy, though if you're interested I would recommend Elements of Information Theory by Cover and Thomas as an introductory text. This concept is introduced pretty early on (chapter 2 I believe). This is the intro text I used in grad school and I thought it did a very good job (granted I had a wonderful instructor as well).
The key thing to pay attention to is that cross-entropy is a function that takes, as input, two probability distributions: q and p and returns a value that is minimal when q and p are equal. q represents an estimated distribution, and p represents a true distribution.
In the context of ML classification we know the actual label of the training data, so the true/target distribution, p, has a probability of 1 for the true label and 0 elsewhere, i.e. p is a one-hot vector.
On the other hand, the estimated distribution (output of a model), q, generally contains some uncertainty, so the probability of any class in q will be between 0 and 1. By training a system to minimize cross entropy we are telling the system that we want it to try and make the estimated distribution as close as it can to the true distribution. Therefore, the class that your model thinks is most likely is the class corresponding to the highest value of q.
Softmax
Again, there are some complicated statistical ways to interpret softmax that we won't discuss here. The key thing from a practical standpoint is that softmax is a function that takes a list of unbounded values as input, and outputs a valid probability mass function with the relative ordering maintained. It's important to stress the second point about relative ordering. This implies that the maximum element in the input to softmax corresponds to the maximum element in the output of softmax.
Consider a softmax activated model trained to minimize cross-entropy. In this case, prior to softmax, the model's goal is to produce the highest value possible for the correct label and the lowest value possible for the incorrect label.
CrossEntropyLoss in PyTorch
The definition of CrossEntropyLoss in PyTorch is a combination of softmax and cross-entropy. Specifically

CrossEntropyLoss(x, y) := H(one_hot(y), softmax(x))

Note that one_hot is a function that takes an index y, and expands it into a one-hot vector.
Equivalently you can formulate CrossEntropyLoss as a combination of LogSoftmax and negative log-likelihood loss (i.e. NLLLoss in PyTorch)

LogSoftmax(x) := ln(softmax(x))
CrossEntropyLoss(x, y) := NLLLoss(LogSoftmax(x), y)

Due to the exponentiation in softmax, there are some computational "tricks" that make directly using CrossEntropyLoss more stable (more accurate, less likely to get NaN), than computing it in stages.
Conclusion
Based on the above discussion the answers to your questions are
1. How to train a "standard" classification network in the best way?
Like the doc says.
2. If the network has a final linear layer, how to infer the probabilities per class?
Apply softmax to the output of the network to infer the probabilities per class. If the goal is to just find the relative ordering or highest probability class then just apply argsort or argmax to the output directly (since softmax maintains relative ordering).
3. If the network has a final softmax layer, how to train the network (which loss, and how)?
Generally, you don't want to train a network that outputs softmaxed outputs for stability reasons mentioned above.
That said, if you absolutely needed to for some reason, you would take the log of the outputs and provide them to NLLLoss
criterion = nn.NLLLoss()
...
x = model(data)    # assuming the output of the model is softmax activated
loss = criterion(torch.log(x), y)

which is mathematically equivalent to using CrossEntropyLoss with a model that does not use softmax activation.
criterion = nn.CrossEntropyLoss()
...
x = model(data)    # assuming the output of the model is NOT softmax activated
loss = criterion(x, y)


Instruction: Calculating gradient from network output in PyTorch gives error

Input:
I am trying to use a manually calculate a gradient using the output of my network, I will then use this in a loss function. I have managed to get an example working in keras, but converting it to PyTorch has proven more difficult
I have a model like:
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(1, 50)
        self.fc2 = nn.Linear(50, 10)
        self.fc3 = nn.Linear(10, 1)

    def forward(self, x):
        x = F.sigmoid(self.fc1(x))
        x = F.sigmoid(self.fc2(x))
        x = self.fc3(x)
        return x

and some data:
x = torch.unsqueeze(torch.linspace(-1, 1, 101), dim=1)
x = Variable(x)

I can then try find a gradient like:
output = net(x)
grad = torch.autograd.grad(outputs=output, inputs=x, retain_graph=True)[0]

I want to be able to find the gradient of each point, then do something like:
err_sqr = (grad - x)**2
loss = torch.mean(err_sqr)**2

However, at the moment if I try to do this I get the error:
grad can be implicitly created only for scalar outputs

I have tried changing the shape of my network output to fix this, but if I change it to much it says its not part of the graph. I can get rid of that error by allowing that, but then it says my gradient is None. I've managed to get this working in keras, so I'm confident that its possible here too, I just need a hand!
My questions are:

Is there a way to "fix" what I have to allow me to calculate the gradient

Output:
PyTorch expects an upstream gradient in the grad call. For usual (scalar) loss functions, the upstream gradient is implicitly assumed to be 1.
You can do a similar thing by passing ones as the upstream gradient:
grad = torch.autograd.grad(outputs=output, inputs=x, grad_outputs=torch.ones_like(output), retain_graph=True)[0]


Instruction: Pytorch 2D Feature Tensor 1D output Same weights across dimension

Input:
I am handeling a timeseries dataset with n timesteps, m features and k objects.
As a result my feature vector has a shape of (n,k,m) While my targets shape is (n,m)
I want to predict the targets for every timestep and object, but with the same weights for every opject. Also my loss function looks like this.
average_loss = loss_func(prediction, labels)
sum_loss = loss_func(sum(prediction), sum(labels))
loss = loss_weight * average_loss + (1-loss_weight) * sum_loss

My plan is to not only make sure, that I predict every item as good as possible, but also that the sum of all items get perdicted. loss_weights is a constant.
Currently I am doing this kind of ugly solution:
features = local_batch.squeeze(dim = 0)
labels = torch.unsqueeze(local_labels.squeeze(dim = 0), 1)
prediction = net(features)

I set my batchsize = 1. And squeeze it to make the k objects my batch.
My network looks like this:
def __init__(self, n_feature, n_hidden, n_output):
    super(Net, self).__init__()
    self.hidden = torch.nn.Linear(n_feature, n_hidden)   # hidden layer
    self.predict = torch.nn.Linear(n_hidden, n_output)   # output layer

def forward(self, x):
    x = F.relu(self.hidden(x))      # activation function for hidden layer
    x = self.predict(x)             # linear output
    return x

How do I make sure I do a reasonable convolution over the opject dimension in order to keep the same weights for all objects, without commiting to batchsize=1? Also, how do I achieve the same loss function, where I compute the loss of the prediction sum vs target sum for any timestamp?

Output:
It's not exactly ugly -- I would do the same but generalize it a bit for batch size >1 using view.
# Using your notations
n, k, m = features.shape
features = local_batch.view(n*k, m)
prediction = net(features).view(n, k, m)

With the prediction in the correct shape (n*k*m), implementing your loss function should not be difficult.


Instruction: How to get the last index of model's prediction?

Input:
I am new to PyTorch. I have a variable pred which has a list of a tensor.
print(pred)
output: [tensor([[176.64380, 193.86154, 273.84702, 306.30405, 0.83492, 2.00000]])]

So I wanted to access the last element which is the class. I did that by first converting the list into a tensor.
x = torch.stack(pred)
output: tensor([[[176.64380, 193.86154, 273.84702, 306.30405, 0.83492, 2.00000]]])

Now, how do I access the last element or is there any better/efficient way of doing this?
EDIT: For further reference here's the code that performs classification task.
def classify_face(image):
    device = torch.device("cpu")
    img = process_image(image)
    print('Image processed')
    # img = image.unsqueeze_(0)
    # img = image.float()

    pred = model(img)[0]

    # Apply NMS
    pred = non_max_suppression(pred, 0.4, 0.5, classes = [0, 1, 2], agnostic = None )
    if classify:
        pred = apply_classifier(pred, modelc, img, im0s)
    #print(pred)

    model.eval()
    model.cpu()

    print(pred)

    # output = non_max_suppression(output, 0.4, 0.5, classes = class_names, agnostic = False)

    #_, predicted = torch.max(output[0], 1)
    #print(predicted.data[0], "predicted")

    classification = torch.cat(pred)[:, -1]
    index = int(classification)
    print(names[index])
    return names[index]

During prediction pred consists of x1, y1, x2, y2, conf, and class.
e.g. pred = [tensor([[176.64380, 193.86154, 273.84702, 306.30405, 0.83492, 2.00000]])]
If there are no predictions made by the model then pred is simply empty.
e.g. pred = [tensor([], size=(0, 6))]
Presently my program stops prediction if it receives an empty tensor and throws an error:
Traceback (most recent call last):
  File "WEBCAM_DETECT.py", line 168, in <module>
    label = classify_face(frame)
  File "WEBCAM_DETECT.py", line 150, in classify_face
    index = int(classification)
ValueError: only one element tensors can be converted to Python scalars

EDIT1:
It seems to work when I check length of the pred but I get this error when there are two or more rows in tensor.
[tensor([[212.38568, 117.47020, 339.35773, 266.00513,   0.74144,   2.00000],
        [214.60651, 118.50694, 339.90192, 265.91696,   0.44277,   0.00000]])]
#################
#################
Traceback (most recent call last):
  File "WEBCAM_DETECT.py", line 172, in <module>
    label = classify_face(frame)
  File "WEBCAM_DETECT.py", line 154, in classify_face
    index = int(classification)
ValueError: only one element tensors can be converted to Python scalars

How do I make my program sort of just ignore if there are no predictions made at a certain frame and continue onto the next frame?

Output:
You can select the last element with the index notation on the 3rd axis, then broadcasting to a 1D tensor:
x[:, :, -1].view(-1)

However, I would rather use torch.cat on pred, this avoids creating a new axis:
torch.cat(pred)[:, -1]

Edit - you may check if the tensor is empty beforehand with:
if len(pred) == 0:
    return None


Instruction: TypeError: 'DiscreteFactor' object is not subscriptable

Input:
I have a Bayesian algorithm program to be executed, I am using python 3
import numpy as np
import csv
import pandas as pd
from pgmpy.models import BayesianModel
from pgmpy.estimators import MaximumLikelihoodEstimator
from pgmpy.inference import VariableElimination

heartDisease = pd.read_csv('heart.csv')
heartDisease = heartDisease.replace('?',np.nan)

print('Few examples from the dataset are given below')
print(heartDisease.head())

model = BayesianModel([('age','trestbps'),('age','fbs'),('sex','trestbps'),('exang','trestbps'),('trestbps','heartdisease'),('fbs','heartdisease'),('heartdisease','restecg'),('heartdisease','thalach'),('heartdisease','chol')])

print('\nLearning CPD using Maximum likelihood estimators')
model.fit(heartDisease,estimator=MaximumLikelihoodEstimator)

print('\n Inferencing with Bayesian Network:')
HeartDisease_infer = VariableElimination(model)

print('\n 1. Probability of HeartDisease given Age=28')
q=HeartDisease_infer.query(variables=['heartdisease'],evidence={'age':28})
print(q['heartdisease'])

print('\n 2. Probability of HeartDisease given cholesterol=100')
q=HeartDisease_infer.query(variables=['heartdisease'],evidence={'chol':100})
print(q['heartdisease'])

the error that i have recieved when i run my Bayesian network program is:
TypeError                                 Traceback (most recent call last)
<ipython-input-7-84a6b48627b2> in <module>
     23 print('\n 1. Probability of HeartDisease given Age=28')
     24 q=HeartDisease_infer.query(variables=['heartdisease'],evidence={'age':28})
---> 25 print(q['heartdisease'])
     26
     27 print('\n 2. Probability of HeartDisease given cholesterol=100')

TypeError: 'DiscreteFactor' object is not subscriptable

So far I haven't seen this exact error here on stackoverflow. Can anyone explain why I am getting this error?

Output:
After trying to solve the error I came up with the solution.
print('\n 1. Probability of HeartDisease given Age=28')
q=HeartDisease_infer.query(variables=['heartdisease'],evidence={'age':28})
print(q['heartdisease'])

print(q['heartdisesase']

over in this part of the code snippet I just removed ['heartdisease'].
here the output was actually trying to store itself into a array object, however the output actually is in special table format that cant be stored into an array so printing the actual answer 'q' gives you a required result.
print(q)

this get's your job done..!!


Instruction: Numpy sum elements in a multi-dimensional array according to indices

Input:
I am dealing with a very large multi-dimensional data , but let me take a 2D array for example. Given a value array that is changing every iteration,
arr = np.array([[ 1, 2, 3, 4, 5], [5, 6, 7, 8, 9]]) # a*b

and an index array that is fixed all the time.
idx = np.array([[[0, 1, 1], [-1, -1, -1]],
                [[5, 1, 3], [1, -1, -1]]]) # n*h*w, where n = a*b,

Here -1 means no index will be applied. And I wish to get a result
res = np.array([[1+2+2, 0],
                [5+2+4, 2]]) # h*w

In real practice, I am doing with a very large 3D tensor (n ~ trillions), with a very sparse idx (i.e. lots of -1). As idx is fixed, my current solution is to pre-compute a n*(h*w) array index_tensor by filling 0 and 1, and then do
tmp = arr.reshape(1, n)
res = (tmp @ index_tensor).reshape([h,w])

It works fine but takes a huge memory to store the index_tensor. Is there any approach that I can take the advantage of the sparsity and unchangeableness of idx to reduce the memory cost and keep a fair running speed in python (using numpy or pytorch would be the best)? Thanks in advance!

Output:
Ignoring the -1 complication for the moment, the straight forward indexing and summation is:
In [58]: arr = np.array([[ 1, 2, 3, 4, 5], [5, 6, 7, 8, 9]])
In [59]: idx = np.array([[[0, 1, 1], [2, 4, 6]],
    ...:                 [[5, 1, 3], [1, -1, -1]]])
In [60]: arr.flat[idx]
Out[60]:
array([[[1, 2, 2],
        [3, 5, 6]],

       [[5, 2, 4],
        [2, 9, 9]]])
In [61]: _.sum(axis=-1)
Out[61]:
array([[ 5, 14],
       [11, 20]])

One way (not necessarily fast or memory efficient) of dealing with the -1 is with a masked array:
In [62]: mask = idx<0
In [63]: mask
Out[63]:
array([[[False, False, False],
        [False, False, False]],

       [[False, False, False],
        [False,  True,  True]]])

In [65]: ma = np.ma.masked_array(Out[60],mask)
In [67]: ma
Out[67]:
masked_array(
  data=[[[1, 2, 2],
         [3, 5, 6]],

        [[5, 2, 4],
         [2, --, --]]],
  mask=[[[False, False, False],
         [False, False, False]],

        [[False, False, False],
         [False,  True,  True]]],
  fill_value=999999)
In [68]: ma.sum(axis=-1)
Out[68]:
masked_array(
  data=[[5, 14],
        [11, 2]],
  mask=[[False, False],
        [False, False]],
  fill_value=999999)

Masked arrays deal with operations like the sum by replacing the masked values with something neutral, such as 0 for the case of sums.
(I may revisit this in the morning).
sum with matrix product
In [72]: np.einsum('ijk,ijk->ij',Out[60],~mask)
Out[72]:
array([[ 5, 14],
       [11,  2]])

This is more direct, and faster, than the masked array approach.
You haven't elaborated on constructing the index_tensor so I won't try to compare it.
Another possibility is to pad the array with a 0, and adjust indexing:
In [83]: arr1 = np.hstack((0,arr.ravel()))
In [84]: arr1
Out[84]: array([0, 1, 2, 3, 4, 5, 5, 6, 7, 8, 9])
In [85]: arr1[idx+1]
Out[85]:
array([[[1, 2, 2],
        [3, 5, 6]],

       [[5, 2, 4],
        [2, 0, 0]]])
In [86]: arr1[idx+1].sum(axis=-1)
Out[86]:
array([[ 5, 14],
       [11,  2]])

sparse
A first stab at using sparse matrix:
Reshape idx to 2d:
In [141]: idx1 = np.reshape(idx,(4,3))

make a sparse tensor from that.  For a start I'll go the iterative lil approach, though usually constructing coo (or even csr) inputs directly is faster:
In [142]: M = sparse.lil_matrix((4,10),dtype=int)
     ...: for i in range(4):
     ...:     for j in range(3):
     ...:         v = idx1[i,j]
     ...:         if v>=0:
     ...:            M[i,v] = 1
     ...:
In [143]: M
Out[143]:
<4x10 sparse matrix of type '<class 'numpy.int64'>'
    with 9 stored elements in List of Lists format>
In [144]: M.A
Out[144]:
array([[1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 1, 0, 1, 0, 1, 0, 0, 0],
       [0, 1, 0, 1, 0, 1, 0, 0, 0, 0],
       [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]])

This can then be used for a sum of products:
In [145]: M@arr.ravel()
Out[145]: array([ 3, 14, 11,  2])

Using M.A@arr.ravel() is essentially what you do.  While M is sparse, arr is not.  For this case M.A@ is faster than M@.


Instruction: Pytorch's nn.TransformerEncoder "src_key_padding_mask" not functioning as expected

Input:
Im working with Pytorch's nn.TransformerEncoder module. I got input samples with (as normal) the shape (batch-size, seq-len, emb-dim). All samples in one batch have been zero-padded to the size of the biggest sample in this batch. Therefore I want the attention of the all zero values to be ignored.
The documentation says, to add an argument src_key_padding_mask to the forward function of the nn.TransformerEncoder module. This mask should be a tensor with shape (batch-size, seq-len) and have for each index either True for the pad-zeros or False for anything else.
I achieved that by doing:
. . .

def forward(self, x):
    # x.size -> i.e.: (200, 28, 200)

    mask = (x == 0).cuda().reshape(x.shape[0], x.shape[1])
    # mask.size -> i.e.: (200, 20)

    x = self.embed(x.type(torch.LongTensor).to(device=device))
    x = self.pe(x)

    x = self.transformer_encoder(x, src_key_padding_mask=mask)

    . . .

Everything works good when I dont set the src_key_padding_mask. But the error I get when I do is the following:
File "/home/me/.conda/envs/py37/lib/python3.7/site-packages/torch/nn/functional.py", line 4282, in multi_head_attention_forward
    assert key_padding_mask.size(0) == bsz
AssertionError

Seems seems like it is comparing the first dimension of the mask, which is the batch-size, with bsz which probably stands for batch-size. But why is it failing then? Help very much appreciated!

Output: I got the same issue, which is not a bug: pytorch's Transformer implementation requires the input x to be (seq-len x batch-size x emb-dim) while yours seems to be (batch-size x seq-len x emb-dim).


Instruction: How to convert one-hot vector to label index and back in Pytorch?

Input:
How to transform vectors of labels to one-hot encoding and back in Pytorch?
The solution to the question was copied to here after having to go through the entire forum discussion, instead of just finding an easy one from googling.

Output:
From the Pytorch forums
import torch
import numpy as np

labels = torch.randint(0, 10, (10,))

# labels --> one-hot
one_hot = torch.nn.functional.one_hot(target)
# one-hot --> labels
labels_again = torch.argmax(one_hot, dim=1)

np.testing.assert_equals(labels.numpy(), labels_again.numpy())


Instruction: Why define a backward method for a custom layer in Pytorch?

Input:
I am currently constructing a model in Pytorch that requires multiple custom layers. I have only been defining the forward method and thus do not define a backward method. The model seems to run well, and the optimizer is able to update using the gradients from the layers. However, I see many people defining backward methods, and I wonder if I am missing something.
Why might you need to define a backwards pass?

Output:
In very few cases should you be implementing your own backward function in PyTorch. This is because PyTorch's autograd functionality takes care of computing gradients for the vast majority of operations.
The most obvious exceptions are

You have a function that cannot be expressed as a finite combination of other differentiable functions (for example, if you needed the incomplete gamma function, you might want to write your own forward and backward which used numpy and/or lookup tables).

You're looking to speed up the computation of a particularly complicated expression for which the gradient could be drastically simplified after applying the chain rule.


Instruction: Defining a Torch Class in R package "torch"

Input:
this post is related to my earlier How to define a Python Class which uses R code, but called from rTorch? .
I came across the torch package in R (https://torch.mlverse.org/docs/index.html) which allows to Define a DataSet class definition. Yet, I also need to be able to define a model class like class MyModelClass(torch.nn.Module) in Python. Is this possible in the torch package in R?
When I tried to do it with reticulate it did not work - there were conflicts like
  ImportError: /User/homes/mreichstein/miniconda3/envs/r-torch/lib/python3.6/site-packages/torch/lib/libtorch_python.so: undefined symbol: _ZTINSt6thread6_StateE

It also would not make much sense, since torch isn't wrapping Python.
But it is loosing at lot of flexibility, which rTorch has (but see my problem in the upper post).
Thanks for any help!
Markus

Output:
You can do that directly using R's torch package which seems quite comprehensive at least for the basic tasks.
Neural networks
Here is an example of how to create nn.Sequential like this:
library(torch)

model <- nn_sequential(
    nn_linear(D_in, H),
    nn_relu(),
    nn_linear(H, D_out)
)

Below is a custom nn_module (a.k.a. torch.nn.Module) which is a simple dense (torch.nn.Linear) layer (source):
library(torch)

# creates example tensors. x requires_grad = TRUE tells that
# we are going to take derivatives over it.
dense <- nn_module(
  clasname = "dense",
  # the initialize function tuns whenever we instantiate the model
  initialize = function(in_features, out_features) {

    # just for you to see when this function is called
    cat("Calling initialize!")

    # we use nn_parameter to indicate that those tensors are special
    # and should be treated as parameters by `nn_module`.
    self$w <- nn_parameter(torch_randn(in_features, out_features))
    self$b <- nn_parameter(torch_zeros(out_features))

  },
  # this function is called whenever we call our model on input.
  forward = function(x) {
    cat("Calling forward!")
    torch_mm(x, self$w) + self$b
  }
)

model <- dense(3, 1)

Another example, using torch.nn.Linear layers to create a neural network this time (source):
two_layer_net <- nn_module(
   "two_layer_net",
   initialize = function(D_in, H, D_out) {
      self$linear1 <- nn_linear(D_in, H)
      self$linear2 <- nn_linear(H, D_out)
   },
   forward = function(x) {
      x %>%
         self$linear1() %>%
         nnf_relu() %>%
         self$linear2()
   }
)

Also there are other resources like here (using flow control and weight sharing).
Other
Looking at the reference it seems most of the layers are already provided (didn't notice transformer layers at a quick glance, but this is minor).
As far as I can tell basic blocks for neural networks, their training etc. are in-place (even JIT so sharing between languages should be possible).


Instruction: Repeat 3d tensor's rows in pytorch

Input:
I have a BxCxd tensor of coordinates and want to repeat each row in the following way:
 [[[1,0,0],[0,1,0],[0,0,1]]] -> [[[1,0,0],[1,0,0],[0,1,0],[0,1,0],[0,0,1],[0,0,1]]]

In the above example each row is repeated 2 times. What's especially important is the ordering. Each row in the first tensor should appear k times in the second one before the next row appears.
I tried the following code:
print(x.size())
params = x.repeat_interleave(self.k, dim=-1).permute(0,2,1)

In the above snippet, x is of size 32x128x4 before repeat_interleave. With self.k = 64 I would expect the result to be a 32x8192x4 tensor, however the result I am getting is 32x256x128 which does not make sense to me. What am I missing here?

Output:
I think you want:
t.repeat_interleave(2, dim=1)

Output:
ensor([[[1, 0, 0],
         [1, 0, 0],
         [0, 1, 0],
         [0, 1, 0],
         [0, 0, 1],
         [0, 0, 1]]])


Instruction: Error when loading torch.hub.load('pytorch/fairseq', 'roberta.large.mnli') on AWS EC2

Input:
I'm trying to run some code using Torch (and Roberta language model) on an EC2 instance on AWS.
The compilation seems to fail, does anyone have a pointer to fix?
Confirm that Torch is correctly installed
import torch
a = torch.rand(5,3)
print (a)

Return this: tensor([[0.7494, 0.5213, 0.8622],...
Attempt to load Roberta
roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli')

Using cache found in /home/ubuntu/.cache/torch/hub/pytorch_fairseq_master
/home/ubuntu/.local/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)
  return torch._C._cuda_getDeviceCount() > 0
fatal: not a git repository (or any of the parent directories): .git
running build_ext
/home/ubuntu/.local/lib/python3.8/site-packages/torch/utils/cpp_extension.py:352: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.
  warnings.warn(msg.format('we could not find ninja.'))
skipping 'fairseq/data/data_utils_fast.cpp' Cython extension (up-to-date)
skipping 'fairseq/data/token_block_utils_fast.cpp' Cython extension (up-to-date)
building 'fairseq.libnat' extension
x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/TH -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/THC -I/usr/include/python3.8 -c fairseq/clib/libnat/edit_dist.cpp -o build/temp.linux-x86_64-3.8/fairseq/clib/libnat/edit_dist.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1011" -DTORCH_EXTENSION_NAME=libnat -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14
In file included from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/ATen/Parallel.h:149,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/utils.h:3,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/nn/cloneable.h:5,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/nn.h:3,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/all.h:12,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/torch.h:3,
                 from fairseq/clib/libnat/edit_dist.cpp:9:
/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/ATen/ParallelOpenMP.h:84: warning: ignoring #pragma omp parallel [-Wunknown-pragmas]
   84 | #pragma omp parallel for if ((end - begin) >= grain_size)

It then ends, after a long while.
x86_64-linux-gnu-gcc: fatal error: Killed signal terminated program cc1plus compilation terminated.
error: command 'x86_64-linux-gnu-gcc' failed with exit status 1

Output:
Got it to work by loading the pretrained model locally instead of from the hub.
from fairseq.models.roberta import RobertaModel
roberta = RobertaModel.from_pretrained('roberta.large.mnli', 'model.pt', '/home/ubuntu/deployedapp/roberta.large')
roberta.eval()

Note that I had to go for a XLarge EC2 instance to run this, otherwise process would be killed due to low memory.


Instruction: Pytorch doesn't work with CUDA in PyCharm/IntelliJ

Input:
I have just downloaded PyTorch with CUDA via Anaconda and when I type into the Anaconda terminal:
import torch
if torch.cuda.is_available():
    print('it works')

then he outputs that;  that means that it worked and it works with PyTorch.
But when I go to my IDE (PyCharm and IntelliJ) and write the same code, it doesn't output anything.
Could someone please explain to me how I can somehow get this to work in the IDE?

Output:
It was driving me mad as well... What finally helped me was the first link that says to use PyCharm "Terminal" to run the pip install command (from the PyTorch website). That fixed all my problems. (I had installed pytorch 3 times by that time and tried different interpreters...)
https://www.datasciencelearner.com/how-to-install-pytorch-in-pycharm/
pip install torch==1.8.0+cu111 torchvision==0.9.0+cu111 torchaudio===0.8.0 -f https://download.pytorch.org/whl/torch_stable.html
I hope this helps save someone hours of headache.   :)


Instruction: RuntimeError: Given groups=1, weight of size [32, 3, 3, 3], expected input[4, 32, 6, 7] to have 3 channels, but got 32 channels instead

Input:
I am trying to implement such CNN.

This is my implementation:
class Net(BaseFeaturesExtractor):
    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 256):
        super(Net, self).__init__(observation_space, features_dim)
        n_input_channels = observation_space.shape[0]
        print("Observation space shape:"+str(observation_space.shape))
        print("Number of channels:" + str(n_input_channels))
        self.cnn = nn.Sequential(
            nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=2, padding=1),
            nn.ReLU(),
            nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=2, padding=1),
            nn.ReLU(),
            nn.Flatten(),
            nn.Linear(in_features=128,out_features=64),
            nn.ReLU(),
            nn.Linear(in_features=64,out_features=7),
            nn.Sigmoid()
        )

    def forward(self, observations: th.Tensor) -> th.Tensor:
        print("Observation shape:"+str(observations[0].shape))
        return self.cnn(observations)

When I tried to run the code which uses this CNN, I am getting following log:
    Observation space shape:(3, 6, 7)
    Number of channels:3
    Observation shape:torch.Size([3, 6, 7])
    Traceback (most recent call last):   File "/Users/joe/Documents/JUPYTER/ConnectX/training3.py", line 250, in <module>
        learner.learn(total_timesteps=iterations, callback=eval_callback)
...
RuntimeError: Given groups=1, weight of size [32, 3, 3, 3], expected input[4, 32, 6, 7] to have 3 channels, but got 32 channels instead

What is the problem here? How can I solve it?

Output:
in_channels of a conv layer should be equal to out_channels of the previous layer. In your case, in_channels of the 2nd and 3rd conv layers don't have the correct values. They should be like below,
    self.cnn = nn.Sequential(
        nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=1, padding=1),
        nn.ReLU(),
        nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1),
        nn.ReLU(),
        ...
    )

Also, you should check in_features of the 1st Linear layer. It depends on the input shape and should be equal to last_conv_out_channels * last_conv_output_height * last_conv_output_width.
For example, for an input=torch.randn(1, 3, 256, 256) last conv layer's output shape would be ([1, 32, 64, 64]), in that case the 1st Linear layer should be,
nn.Linear(in_features=32*64*64,out_features=64)

---- Update after the comment:
Output shape of a conv layer is calculated through the formula here (see under "Shape:" section). Using input = torch.randn(1, 3, 256, 256) as input to the network, here are outputs of each conv layer (I skipped the ReLUs since they don't change the shape),
conv1: (1, 3, 256, 256) -> (1, 32, 256, 256)
conv2: (1, 32, 256, 256) -> (1, 32, 128, 128)
conv3: (1, 32, 128, 128) -> (1, 32, 64, 64)

So how did last_conv_output_height and last_conv_output_width became 64 ? The last conv layer is defined as follows,
nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1)

Data is processed as (num_samples, num_channels, height, width) in PyTorch and the default value for dilation is stated as 1 in the conv2d doc. So, for the last conv layer, H_in is 128, padding[0] is 1, dilation[0] is 1, kernel_size[0] is 3 and stride[0] is 2. Therefore, height of its output becomes,
H_out = ⌊(128 + 2 * 1 - 1 * (3 - 1) - 1) / 2⌋ + 1
H_out = 64

Since square-size kernels and equal-size stride, padding and dilation are used, W_out also becomes 64 for the last conv layer.
I think the easiest way to compute in_features for the 1st Linear layer would be run the model for the desired size input until that layer. An example for your architecture,
inp = torch.randn(1, 3, 256, 256)
arch = nn.Sequential(
    nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1),
    nn.ReLU(),
    nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1),
    nn.ReLU(),
    nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1)
)
outp = arch(inp)
print('outp.shape:', outp.shape)

This prints,
outp.shape: torch.Size([1, 32, 64, 64])

Finally, last_conv_out_channels is out_channels of the last conv layer. The last conv layer in your architecture is nn.Conv2d(32, 32, kernel_size=3, stride=2, padding=1). Here out_channels is the 2nd parameter, so last_conv_out_channels is 32.


Instruction: pytorch custom loss function on minimizing the angle between vectors

Input:
The goal is to minimize the angle between the actual and predicted vectors in a neural network setting. Can someone please check if the following execution is correct?
criterion = nn.CosineSimilarity()
loss = torch.mean(torch.abs(criterion(actual_vectors,predicted_vectors)))

#back-propagation on the above *loss* will try cos(angle) = 0. But I want angle between the vectors to be 0 or cos(angle) = 1.

loss = 1 - loss

#To me, the above does not seem right. Isn't back-propagation on the above 'loss' similar to minimizing the negative of 'loss' from line 2?
#Does '1' have no role to play here when back-propagation is applied?

loss.backward()

Output:
Theoretically that makes sense. The goal of back-propagation is to minimize the loss. If the loss is 1 - cos(A) (where A is the angle difference between the two) then that is equivalent to saying that the goal is to maximize cos(A), which in turn is equivalent to minimizing the Angle between the two vectors.
A simple example would be the goal of minimizing X^2 + 4 the answer to that optimization problem is the same as the answer to the goal of maximizing -(X^2 + 4). Sticking a minus on the whole equation and swapping min with max would make the statements equivalent. So if you have a function you want to MAXIMIZE and your optimization model can only MINIMIZE then just slap a minus sign on your function and call it a day.
Another question you might ask is "what is significant about the 1? Could we have just said loss = -loss" and the answer is... it depends. Theoretically yes that is equivalent and the one doesn't play a role in the backward propagation (since it disappears with the derivative). However, once we start talking about actual optimization with numerical errors and complicated optimizers/update rules then the constant 1 might play a role.
Another reason to have the 1 is so that your loss is nicely defined between 0 and 1 which is a nice property to have.
So yes, minimizing the loss of 1 - cos(A) through back-propagation is equivalent to minimizing the angle between the vectors.


Instruction: what is the equivalent sytax for `Tensor.grad` in Tensorflow 2.0

Input:
In the Pytorch, we can access the gradient of a variable  x by
z.grad

What is the same sytax in Tensorflow 2. My goad is to cut the gradient. Here is Pytorch code
if z.grad > 1000:
    z.grad = 10

Can tensorflow 2 apply the same functions?
Thanks

Output:
So in TF2, assume we define following variables and optimizer:
import tensorflow as tf
from tensorflow import keras

opt = tf.keras.optimizers.Adam(learning_rate=0.1)

x = tf.Variable([3.0, 4.0])
y = tf.Variable([1.0, 1.0, 1.0, 1.0])
var_list = [x, y]

Then we can get gradients by using tf.GradientTape():
with tf.GradientTape() as tape:
    loss = tf.reduce_sum(x ** 2) + tf.reduce_sum(y)

grads = tape.gradient(loss, var_list)

Finally we could process the gradients by custom function:
def clip_grad(grad):
    if grad > 1000:
        grad = 10
    return grad

processed_grads = [tf.map_fn(clip_grad, g) for g in grads]

opt.apply_gradients(zip(processed_grads, var_list))

Note you may find the keras optimizers have get_gradients method, but it won't work with eager execution enabled which is default in TF2, if you want use that, then you may have to write code in TF1 fashion


Instruction: What is the difference between an Embedding Layer with a bias immediately afterwards and a Linear Layer in PyTorch

Input:
I am reading the "Deep Learning for Coders with fastai & PyTorch" book. I'm still a bit confused as to what the Embedding module does. It seems like a short and simple network, except I can't seem to wrap my head around what Embedding does differently than Linear without a bias. I know it does some faster computational version of a dot product where one of the matrices is a one-hot encoded matrix and the other is the embedding matrix. It does this to in effect select a piece of data? Please point out where I am wrong. Here is one of the simple networks shown in the book.
class DotProduct(Module):
    def __init__(self, n_users, n_movies, n_factors):
        self.user_factors = Embedding(n_users, n_factors)
        self.movie_factors = Embedding(n_movies, n_factors)

    def forward(self, x):
        users = self.user_factors(x[:,0])
        movies = self.movie_factors(x[:,1])
        return (users * movies).sum(dim=1)

Output:
Embedding

[...] what Embedding does differently than Linear without a bias.

Essentially everything. torch.nn.Embedding is a lookup table; it works the same as torch.Tensor but with a few twists (like possibility to use sparse embedding or default value at specified index).
For example:
import torch

embedding = torch.nn.Embedding(3, 4)

print(embedding.weight)

print(embedding(torch.tensor([1])))

Would output:
Parameter containing:
tensor([[ 0.1420, -0.1886,  0.6524,  0.3079],
        [ 0.2620,  0.4661,  0.7936, -1.6946],
        [ 0.0931,  0.3512,  0.3210, -0.5828]], requires_grad=True)
tensor([[ 0.2620,  0.4661,  0.7936, -1.6946]], grad_fn=<EmbeddingBackward>)

So we took the first row of the embedding. It does nothing more than that.
Where is it used?
Usually when we want to encode some meaning (like word2vec) for each row (e.g. words being close semantically are close in euclidean space) and possibly train them.
Linear
torch.nn.Linear (without bias) is also a torch.Tensor (weight) but it does operation on it (and the input) which is essentially:
output = input.matmul(weight.t())

every time you call the layer (see source code and functional definition of this layer).
Code snippet
The layer in your code snippet does this:

creates two lookup tables in __init__
the layer is called with input of shape (batch_size, 2):

first column contains indices of user embeddings
second column contains indices of movie embeddings

these embeddings are multiplied and summed returning (batch_size,) (so it's different from nn.Linear which would return (batch_size, out_features) and perform dot product instead of element-wise multiplication followed by summation like here)

This is probably used to train both representations (of users and movies) for some recommender-like system.
Other stuff

I know it does some faster computational version of a dot product
where one of the matrices is a one-hot encoded matrix and the other is
the embedding matrix.

No, it doesn't. torch.nn.Embedding can be one hot encoded and might also be sparse, but depending on the algorithms (and whether those support sparsity) there might be performance boost or not.


Instruction: How To Get The Pixel Count Of A Segmented Area in an Image I used Vgg16 for Segmentation

Input: I am new to deep learning but have succeeded in semantic segmentation of the image I am trying to get the pixel count of each class in the label. As an example in the image I want to get the pixel count of the carpet, or the chandelier or the light stand. How do I go about? Thanks any suggestions will help.

Output:
Edit: In what format the regions are returned? Do you have only the final image or the regions are given as contours? If you have them as contours (list of coordinates), you can apply findContourArea directly on that structure.

If you can receive/sample the regions one by one in an image (but do not have the contour), you can sequentially paint each of the colors/classes in a clear image, either convert it to grayscale or directly paint it in grayscale or binary, or binarize with threshold; then numberPixels = len(cv2.findNonZero(bwImage)). cv2.findContour and cv2.contourArea should do the same.

Instead of rendering each class in a separate image, if your program receives only the final segmentation and not per-class contours, you can filter/mask the regions by color ranges on that image. I built that and it seemed to do the job, 14861 pixels for the pink carpet:

import cv2
import numpy as np

# rgb 229, 0, 178 # the purple carpet in RGB (sampled with IrfanView)
# b,g,r = 178, 0, 229 # cv2 uses BGR
class_color = [178, 0, 229]
multiclassImage = cv2.imread("segmented.png")
cv2.imshow("MULTI", multiclassImage)
filteredImage = multiclassImage.copy()
low = np.array(class_color);

mask = cv2.inRange(filteredImage, low, low)
filteredImage[mask == 0] = [0, 0, 0]
filteredImage[mask != 0] = [255,255,255]
cv2.imshow("FILTER", filteredImage)
# numberPixelsFancier = len(cv2.findNonZero(filteredImage[...,0]))
# That also works and returns 14861 - without conversion, taking one color channel
bwImage = cv2.cvtColor(filteredImage, cv2.COLOR_BGR2GRAY)
cv2.imshow("BW", bwImage)
numberPixels = len(cv2.findNonZero(bwImage))
print(numberPixels)
cv2.waitKey(0)

If you don't have the values of the colors given or/and can't control them, you can use numpy.unique(): https://numpy.org/doc/stable/reference/generated/numpy.unique.html and it will return the unique colors, then they could be applied in the algorithm above.

Edit 2: BTW, another way to compute or verify such counts is by calculating histograms. That's with IrfanView on the black-white image:


Instruction: Predictions using Logistic Regression in Pytorch return infinity

Input:
I started watching a tutorial on PyTorch and I am learning the concept of logistic regression.
I tried it using some stock data that I had. I have inputs, which contains two parameters trade_quantity and trade_value, and targets which has the corresponding stock price.
inputs = torch.tensor([[182723838.00, 2375432.00],
                       [185968153.00, 2415558.00],
                       [181970093.00, 2369140.00],
                       [221676832.00, 2811589.00],
                       [339785916.00, 4291782.00],
                       [225855390.00, 2821301.00],
                       [151430199.00, 1889032.00],
                       [122645372.00, 1552998.00],
                       [129015052.00, 1617158.00],
                       [121207837.00, 1532166.00],
                       [139554705.00, 1789392.00]])

targets = torch.tensor([[76.90],
                        [76.90],
                        [76.90],
                        [80.70],
                        [78.95],
                        [79.60],
                        [80.05],
                        [78.90],
                        [79.40],
                        [78.95],
                        [77.80]])

I defined the model function, the loss as the mean square error, and tried to run it a few times to get some predictions. Here's the code:
def model(x):
    return x @ w.t() + b

def mse(t1, t2):
    diff = t1 - t2
    return torch.sum(diff * diff) / diff.numel()

preds = model(inputs)
loss = mse(preds, targets)
loss.backward()
with torch.no_grad():
    w -= w.grad * 1e-5
    b -= b.grad * 1e-5
    w.grad.zero_()
    b.grad.zero_()

I am using Jupyter for this and ran the last part of the code a few times, after which the predictions come as:
tensor([[inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf],
        [inf]], grad_fn=<AddBackward0>)

If I run it for a few more times the predictions become nan. Can you please tell me why is this happening?

Output:
To me, this looks more like linear regression than logistic regression. You are trying to fit a linear model onto your data. It's different to a binary classification task where you would need to use a special kind of activation function (a sigmoid for instance) so that the output is either 0 or 1.
In this particular instance you want to solve a 2D linear problem given input x of shape (batch, x1, x2) (where x1 is trade_quantity and x2 is trade_value) and target (batch, y) (y being the stock_price).
So the objective is to find the best w and b matrices (weight matrix and bias column) so that x@w + b is the closest to y as possible, according to your criterion, the mean square error.

I would recommend normalizing your data so it stays in a [0, 1] range. You can do so by measuring the mean and standard deviation of inputs and targets.
inputs_min, inputs_max = inputs.min(axis=0).values, inputs.max(axis=0).values
targets_min, targets_max = targets.min(axis=0).values, targets.max(axis=0).values

Then applying the transformation:
x = (inputs - inputs_min)/(inputs_max - inputs_min)
y = (targets - targets_min)/(targets_max - targets_min)

Try changing your learning rate and have it run for multiple epochs.
lr = 1e-2
for epochs in range(100):
    preds = model(x)
    loss = mse(preds, y)
    loss.backward()
    with torch.no_grad():
        w -= lr*w.grad
        b -= lr*b.grad
        w.grad.zero_()
        b.grad.zero_()

I use a (1, 2) randomly initialized matrix for w (and a (1,) matrix for b):
w = torch.rand(1, 2)
w.requires_grad = True
b = torch.rand(1)
b.requires_grad = True

And got the following train loss over 100 epochs:

To find the right hyperparameters, it's better to have a validation set. This set will get normalized with the mean and std from the train set. It will be used to evaluate the performances at the end of each epoch on data that is 'unknown' to the model. Same goes for your test set, if you have one.


Instruction: where is torch.tensor.item() defined at the Github?

Input:
I am torch Newbie.
when I was learning torch tutorial like below,
I was curious about where tensor.item() is defined.
import torch
a = torch.tensor([1])
print( a.item() )  # it works without problem.

In order to find what i dont know,
First, I used VScode. but I got this.
enter image description here
not like this
enter image description here
Second, I searched "def item()" at the torch Github
But, I could not find. T^T
Could you tell me where tensor.item() is defined at the Github?
Or
where Class_torch.tensor's Menber functions(Method function) are defined??

Output:
Short answer:
torch/csrc/autograd/python_variable_indexing.cpp:268
Long answer:
I hope you like C++. ;)
The first thing to know is that item() isn't (usually) a method in most Python classes. Instead, Python converts calls to item() into other underlying methods like __getitem__() as a convenience. Knowing that:
class Tensor is defined at torch/tensor.py:40.
Most of Torch's underlying, compute intensive functionality is implemented in C and C++, including Tensor. "class Tensor" is based on Torch._C.TensorBase which is provided to Python via a C API from torch/csrc/autograd/python_variable.cpp:812
THPVariableType is a map given to Python describing the C++ functions available on the object from Python. It's defined at torch/csrc/autograd/python_variable. The part relevant to you is the tp_as_mapping entry (line 752), which provides functions for objects implementing the mapping protocol - basically Pyton Array-like objects (Python Documentation)
The THPVariable_as_mapping structure at line 725 provides the mapping methods.The second variable provides the a subscript function used to get items by index (Python documentation)
Therefore, The C++ function TPHVariable_getitem provides the implementation of Torch.Tensor.item(), and it is defined at torch/csrc/autograd/python_variable_indexing


Instruction: PyTorch : How to apply the same random transformation to multiple image?

Input:
I am writing a simple transformation for a dataset which contains many pairs of images. As a data augmentation, I want to apply some random transformation for each pair but the images in that pair should be transformed in the same way.
For example, given a pair of two images A and B, if A is flipped horizontally, B must be flipped horizontally as A. Then the next pair C and D should be differently transformed from A and B but C and D are transformed in the same way. I am trying that in the way below
import random
import numpy as np
import torchvision.transforms as transforms
from PIL import Image

img_a = Image.open("sample_ajpg") # note that two images have the same size
img_b = Image.open("sample_b.png")
img_c, img_d = Image.open("sample_c.jpg"), Image.open("sample_d.png")

transform = transforms.RandomChoice(
    [transforms.RandomHorizontalFlip(),
     transforms.RandomVerticalFlip()]
)
random.seed(0)
display(transform(img_a))
display(transform(img_b))

random.seed(1)
display(transform(img_c))
display(transform(img_d))

Yet、 the above code does not choose the same transformation and as I tested, it is dependent on the number of times transform is called.
Is there any way to force transforms.RandomChoice to use the same transform when specified?

Output:
Usually a workaround is to apply the transform on the first image, retrieve the parameters of that transform, then apply with a deterministic transform with those parameters on the remaining images. However, here RandomChoice does not provide an API to get the parameters of the applied transform since it involves a variable number of transforms.
In those cases, I usually implement an overwrite to the original function.
Looking at the torchvision implementation, it's as simple as:
class RandomChoice(RandomTransforms):
    def __call__(self, img):
        t = random.choice(self.transforms)
        return t(img)

Here are two possible solutions.

You can either sample from the transform list on __init__ instead of on __call__:
import random
import torchvision.transforms as T

class RandomChoice(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.t = random.choice(self.transforms)

    def __call__(self, img):
        return self.t(img)

So you can do:
transform = T.RandomChoice([
     T.RandomHorizontalFlip(),
     T.RandomVerticalFlip()
])
display(transform(img_a)) # both img_a and img_b will
display(transform(img_b)) # have the same transform

transform = T.RandomChoice([
    T.RandomHorizontalFlip(),
    T.RandomVerticalFlip()
])
display(transform(img_c)) # both img_c and img_d will
display(transform(img_d)) # have the same transform

Or better yet, transform the images in batch:
import random
import torchvision.transforms as T

class RandomChoice(torch.nn.Module):
    def __init__(self, transforms):
       super().__init__()
       self.transforms = transforms

    def __call__(self, imgs):
        t = random.choice(self.transforms)
        return [t(img) for img in imgs]

Which allows to do:
transform = T.RandomChoice([
     T.RandomHorizontalFlip(),
     T.RandomVerticalFlip()
])

img_at, img_bt = transform([img_a, img_b])
display(img_at) # both img_a and img_b will
display(img_bt) # have the same transform

img_ct, img_dt = transform([img_c, img_d])
display(img_ct) # both img_c and img_d will
display(img_dt) # have the same transform


Instruction: How to handle this case in pytorch?

Input:
I have the following kinds of two corelated tensors data and mask. The size of data is 1x2x24x2 and the size of mask is 1x2x24. In mask, the True means the corresponding data in data is efficient and should be gradient back-propagated.
data:
tensor([[[[ 1.0663e+03,  5.5993e+02],
          [ 1.0612e+03,  7.2023e+02],
          [ 1.0831e+03,  7.2179e+02],
          [ 1.0945e+03,  5.6083e+02],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10]],

         [[ 6.9314e+02,  1.9700e+02],
          [ 6.3300e+02,  2.6924e+02],
          [ 6.3300e+02,  3.4165e+02],
          [ 7.7515e+02,  4.6000e+02],
          [ 8.2805e+02,  4.6000e+02],
          [ 9.0900e+02,  3.6276e+02],
          [ 9.0900e+02,  2.9035e+02],
          [ 7.9688e+02,  1.9700e+02],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10],
          [-1.0000e+10,  1.0000e+10]]]])
torch.Size([1, 2, 24, 2])

mask:
tensor([[[ True,  True,  True,  True, False, False, False, False, False, False,
          False, False, False, False, False, False, False, False, False, False,
          False, False, False, False],
         [ True,  True,  True,  True,  True,  True,  True,  True, False, False,
          False, False, False, False, False, False, False, False, False, False,
          False, False, False, False]]])
torch.Size([1, 2, 24])

Based on the data and mask, I need further processing. For example, I need to do this kind of for loop:
B, N = data.shape[0:2]
result = torch.zeros((B, N), dtype=torch.float32)
for b in range(B):
    for n in range(N):
        tmp_data = data[b,n]  # 24 x 2
        tmp_mask = mask[b,n]  # 24
        selected = tmp_data[tmp_mask]
        selected = torch.cat([selected, selected[0][None]], dim=0)
        total = selected[0:-1, 0] * selected[1:, 1] - selected[0:-1, 1] * selected[1:, 0]   # do cross product
        total = torch.sum(total, dim=0)
        result[b,n] = torch.abs(total) / 2

Then, the result tensor is fed into subsequent processing, e.g. loss computation. My question is, is there any speed up method to get rid of the for-loop? The main difficulty lies in that for each b and n, the count of True in mask is not equal, sometimes it is 3, sometimes it is 5, but the maximum count of True in each sample is 8. So, in the for-loop, it is necessary to handle each case one by one. Is there any vectorized way for this?
Another difficulty lies in the gradient back-propagation, which I need further investigation to check if this routine can work correctly for gradient back-propagation.
I'm looking forward to an elegant way to handle this kind of tensor processing in pytorch. Thanks in advance!
UPDATED:
The data and mask were sorted by the previous routine, for example:
sorted_inds = ...  # B x N x 24

data = torch.gather(data_original, dim=2, index=sorted_indices.unsqueeze(-1).repeat(1,1,1,2))  # B x N x 24 x 2
mask= torch.gather(mask_original, dim=2, index=sorted_indices)  # B x N x 24

# then the data and mask are obtained
# continue processing in this post
...

Output:
You are looking to vectorize the two for loops with tensor operations. Generally speaking this is possible by just ignoring the first two axes with performing operations on the last two. However, in in this particular case I don't think this is achievable.
The reason lies with the masking that takes place on data with mask (via indexing). This operation will only return a flattened tensor:.
data[mask]
tensor([[1066.3000,  559.9300],
        [1061.2000,  720.2300],
        [1083.1000,  721.7900],
        [1094.5000,  560.8300],
        [ 693.1400,  197.0000],
        [ 633.0000,  269.2400],
        [ 633.0000,  341.6500],
        [ 775.1500,  460.0000],
        [ 828.0500,  460.0000],
        [ 909.0000,  362.7600],
        [ 909.0000,  290.3500],
        [ 796.8800,  197.0000]])

As you can see the correct values are retained but not the desired shape. That's where the problem lies. This is what we would have wished to have (in order to proceed with a vectorized form). This is not possible as all tensor on a dimension must have the same length: the size on that dimension.
tensor([[[1066.3000,  559.9300],
         [1061.2000,  720.2300],
         [1083.1000,  721.7900],
         [1094.5000,  560.8300]],
        [[ 693.1400,  197.0000],
         [ 633.0000,  269.2400],
         [ 633.0000,  341.6500],
         [ 775.1500,  460.0000],
         [ 828.0500,  460.0000],
         [ 909.0000,  362.7600],
         [ 909.0000,  290.3500],
         [ 796.8800,  197.0000]])

Concerning the cross-product. PyTorch provides a function for that: torch.cross, but it can only be used for 3D tensors. Above that it is not possible to use it. I think your implementation is fine.

Some suggestions, these are not groundbreaking improvements but can help:

There's a trick with indexing to avoid having to do x.unsqueeze(0) (or as you did x[None]) on one-element tensors: by slicing x[:1].

Argument dim generally defaults to 0. For .sum() you don't need it since you are looking for the total sum and not a sum per-axis.

You could flatten the first two axes, perform a single loop and reshape at the end.

Here, I have taken the liberty of tweaking your code:
B, N, *_ = data.shape
data_ = data.reshape(B*N, -1, 2)
mask_ = mask.reshape(B*N, -1)

result = []
for i in range(B*N):
    selected = data_[i][mask_[i]]
    looped = torch.cat([selected, selected[:1]])
    cross = looped[:-1, 0]*looped[1:, 1] - looped[:-1, 1]*looped[1:, 0]
    result.append(cross.sum().abs() / 2)

torch.tensor(result).reshape(B, N)


Instruction: BigBird, or Sparse self-attention: How to implement a sparse matrix?

Input:
This question is related to the new paper: Big Bird: Transformers for Longer Sequences. Mainly, about the implementation of the Sparse Attention (that is specified in the Supplemental material, part D). Currently, I am trying to implement it in PyTorch.
They suggest a new way to speed up the computation by blocking the original query and key matrices (see, below)
When you do the matrix multiplaciton in the step (b), you end up with something like that:
.
So I was wondering: how would you go from that representation (image above) to a sparse matrix (using PyTorch, see below)? In the paper, they just say: "simply reshape the result", and I do not know any easy ways to do so (especially, when I have multiple blocks in different positions (see step (c) on the first image).

RESOLUTION:
Huggingface has an implementation of BigBird in pytorch.

Output:
I end up following the guidelines in the paper. When it comes to the unpacking of the result I use: torch.sparse_coo_tensor
EDIT: Sparse tensors are still memory-hungry! The more efficient solution is described here


Instruction: What is the opacity graph in tensorboard?

Input:
I am new to Tensorboard.
This is an output I have by this line:
self.logger.experiment.add_scalars("losses", {"train_loss": loss}, global_step=self.current_epoch)

I can only touch the dark blue line, which is probably the loss I logged, but not the light blue line.

What is the light blue line?
Why is the dark blue line not strictly continuous?

Output:
I asked the same question on Github.

The light one is the actual data, and the bold curve is the smoothed
curve. Use the slider in the UI to adjust it.


Instruction: Pytorch: Why batch is the second dimension in the default LSTM?

Input:
In the PyTorch LSTM documentation it is written:

batch_first – If True, then the input and output tensors are provided
as (batch, seq, feature). Default: False

I'm wondering why they chose the default batch dimension as the second one and not the first one. for me, it is easier to imaging my data as [batch, seq, feature] than [seq, batch, feature]. The first one is more intuitive for me and the second one is counterintuitive.
I'm asking here to know if the is any reason behind this and if you can help me to have some understanding about it.

Output:
As far as I know, there is not a heavily justified answer. Nowadays, it is different from other frameworks where, as you say, the shape is more intuitive, such as Keras, but only for compatibility reasons with older versions, changing a default parameter that modifies the dimensions of a vector would probably break half of the models out there if their maintainers update to newer PyTorch versions.
Probably the idea, in the beginning, was to set the temporal dimension first to simplify the iterating process over time, so you can just do a
for t, out_t in enumerate(my_tensor)

instead of having to do less visual stuff such as accessing with my_tensor[:, i] and having to iterate in range(time).


Instruction: why pytorch linear model isn't using sigmoid function

Input:
I'm expecting that the linear model in pytorch yields sigmoid(WX+b). But I see it is only returning Wx+b. Why is this the case?
In Udacity "Intro to deep learning with pytorch" -> Lesson 2: Introduction to Neural Networks, They say that the output is sigmoid:
̂ =(11+22+)

From the below code I was expecting y cap to be 0.38391371665752183, but it is just the value of WX+b, which I confirmed the output. Why is this discrepancy?
import torch
from torch import nn
import numpy as np

torch.manual_seed(0)
model = nn.Linear(2,1)
w1 = model.weight.detach().numpy()
b1 = model.bias.detach().numpy()
print (f'model.weight = {w1}, model.bias={b1}')
x = torch.tensor([[0.2877, 0.2914]])
print(f'model predicted {model(x)}')
z = x.numpy()[0][0] * w1[0][0] + x.numpy()[0][1] * w1 [0][1] + b1[0]
print(f'manual multiplication yielded {z}')
ycap = 1/(1+ np.exp(-z))
print(f'y cap is {ycap}')

Output:
model.weight = [[-0.00529398  0.3793229 ]], model.bias=[-0.58198076]
model predicted tensor([[-0.4730]], grad_fn=<AddmmBackward>)
manual multiplication yielded -0.4729691743850708
y cap is 0.38391371665752183

Output:
The nn.Linear layer is a linear fully connected layer. It corresponds to wX+b, not sigmoid(WX+b).
As the name implies, it's a linear function. You can see it as a matrix multiplication (with or without a bias). Therefore it does not have an activation function (i.e. nonlinearities) attached.
If you want to append an activation function to it, you can do so by defining a sequential model:
model = nn.Sequential(
    nn.Linear(2, 1)
    nn.Sigmoid()
)

Edit - if you want to make sure:
x = torch.tensor([[0.2877, 0.2914]])
model = nn.Linear(2,1)
m1 = nn.Sequential(model, nn.Sigmoid())

m1(x)[0].item(), torch.sigmoid(model(x))[0].item()