Instruction: Runtimeerror: Cuda out of memory - problem in code or gpu?

Input:
I am currently working on a computer vision project. I keep getting a runtime error that says "CUDA out of memory". I have tried all possible ways like reducing batch size and image resolution, clearing the cache, deleting variables after training starts, reducing image data and so on... Unfortunately, this error doesn't stop. I have a Nvidia Geforce 940MX graphics card on my HP Pavilion laptop. I have installed cuda 10.2 and cudNN from the pytorch installation page. My aim was to create a flask website out of this model but I am stuck with this issue. Any suggestions to this problem will be helpful.
This is my code
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import os
import cv2
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import albumentations as A
from torch.utils.data import TensorDataset, DataLoader,Dataset
from torchvision import models
from collections import defaultdict
from torch.utils.data.sampler import RandomSampler
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn import model_selection
from tqdm import tqdm
import gc

# generate data from csv file
class Build_dataset(Dataset):

    def __init__(self, csv, split, mode, transform=None):
        self.csv = csv.reset_index(drop=True)
        self.split = split
        self.mode = mode
        self.transform = transform

    def __len__(self):
        return self.csv.shape[0]

    def __getitem__(self, index):
        row = self.csv.iloc[index]

        image = cv2.imread(row.filepath)
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image=image)
            image = res['image'].astype(np.float32)
        else:
            image = image.astype(np.float32)

        image = image.transpose(2, 0, 1)
        data = torch.tensor(image).float()

        if self.mode == 'test':
            return data
        else:
            return data, torch.tensor(self.csv.iloc[index].target).long()

# training epoch
def train_epoch(model, loader, optimizer,loss_fn,device, scheduler,n_examples):

model = model.train()

losses = []
correct_predictions = 0

for inputs, labels in tqdm(loader):
    inputs = inputs.to(device)
    labels = labels.to(device)

    outputs = model(inputs)

    _, preds = torch.max(outputs, dim=1)
    loss = loss_fn(outputs, labels)

    correct_predictions += torch.sum(preds == labels)
    losses.append(loss.item())

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()
# here you delete inputs and labels and then use gc.collect
    del inputs, labels
    gc.collect()

return correct_predictions.double() / n_examples, np.mean(losses)

# validation epoch
def val_epoch(model, loader,loss_fn, device,n_examples):

model = model.eval()

losses = []
correct_predictions = 0

with torch.no_grad():
    for inputs, labels in tqdm(loader):
        inputs = inputs.to(device)
        labels = labels.to(device)
        outputs = model(inputs)
        _, preds = torch.max(outputs, dim=1)
        loss = loss_fn(outputs, labels)
        correct_predictions += torch.sum(preds == labels)
        losses.append(loss.item())
        # here you delete inputs and labels and then use gc.collect
        del inputs, labels
        gc.collect()

return correct_predictions.double() / n_examples, np.mean(losses)

def train(model,device, num_epochs):
# generate data
dataset_train = Build_dataset(df_train,  'train', 'train', transform=transforms_train)
dataset_valid = Build_dataset(df_valid, 'train', 'val', transform=transforms_val)

#load data
train_loader = DataLoader(dataset_train, batch_size = 16,sampler=RandomSampler(dataset_train), num_workers=4)
valid_loader = DataLoader(dataset_valid, batch_size = 16,shuffle = True, num_workers= 4 )

dataset_train_size = len(dataset_train)

dataset_valid_size = len(dataset_valid)

optimizer = optim.Adam(model.parameters(), lr = 3e-5)

model = model.to(device)

scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, patience = 3,threshold = 0.001, mode = 'max')

loss_fn = nn.CrossEntropyLoss().to(device)

history = defaultdict(list)

best_accuracy = 0.0

for epoch in range(num_epochs):
    print(f'Epoch {epoch+1} / {num_epochs}')
    print ('-'*30)

    train_acc, train_loss = train_epoch(model, train_loader, optimizer, loss_fn, device, scheduler, dataset_train_size)
    print(f'Train loss {train_loss} accuracy {train_acc}')
    valid_acc, valid_loss = val_epoch(model, valid_loader, loss_fn, device,dataset_valid_size)
    print(f'Val   loss {valid_loss} accuracy {valid_acc}')
    print()

    history['train_acc'].append(train_acc)
    history['train_loss'].append(train_loss)
    history['val_acc'].append(valid_acc)
    history['val_loss'].append(valid_loss)

    if valid_acc > best_accuracy:
        torch.save(model.state_dict(), 'best_model_state.bin')
        best_accuracy = valid_acc

print('Best Accuracy: {best_accuracy}')

model.load_state_dict(torch.load('best_model_state.bin'))

return model, history

if __name__ == '__main__':
#competition data -2020
data_dir = "C:\\Users\\Aniruddh\\Documents\\kaggle\\jpeg_melanoma_2020"
#competition data - 2019
data_dir2 = "C:\\Users\\Aniruddh\\Documents\\kaggle\\jpeg_melanoma_2019"
# device
device = torch.device("cuda")

# augmenting images

image_size = 384
transforms_train = A.Compose([
    A.Transpose(p=0.5),
    A.VerticalFlip(p=0.5),
    A.HorizontalFlip(p=0.5),
    A.RandomBrightness(limit=0.2, p=0.75),
    A.RandomContrast(limit=0.2, p=0.75),
    A.OneOf([
        A.MedianBlur(blur_limit=5),
        A.GaussianBlur(blur_limit=5),
        A.GaussNoise(var_limit=(5.0, 30.0)),
    ], p=0.7),

    A.OneOf([
        A.OpticalDistortion(distort_limit=1.0),
        A.GridDistortion(num_steps=5, distort_limit=1.),
        A.ElasticTransform(alpha=3),
    ], p=0.7),

    A.CLAHE(clip_limit=4.0, p=0.7),
    A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),
    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),
    A.Resize(image_size, image_size),
    A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1, p=0.7),
    A.Normalize()
])

transforms_val = A.Compose([
    A.Resize(image_size, image_size),
    A.Normalize()
])
# create data
df_train = pd.read_csv("C:\\Users\\Aniruddh\\Documents\\kaggle\\jpeg_melanoma_2020\\train.csv")  #/kaggle/input/siim-isic-melanoma-classification/train.csv
df_train.head()

df_train['is_ext'] = 0
df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir, 'train', f'{x}.jpg'))

# dataset from 2020 data
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis', 'BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis', 'BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo', 'BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS', 'BKL'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule', 'unknown'))
df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation', 'unknown'))

# dataset from 2019 data
df_train2 = pd.read_csv('/content/drive/My Drive/siim_melanoma images/train_2019.csv')
df_train2 = df_train2[df_train2['tfrecord'] >= 0].reset_index(drop=True)
#df_train2['fold'] = df_train2['tfrecord'] % 5
df_train2['is_ext'] = 1
df_train2['filepath'] = df_train2['image_name'].apply(lambda x: os.path.join(data_dir2, 'train', f'{x}.jpg'))

df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('NV', 'nevus'))
df_train2['diagnosis'] = df_train2['diagnosis'].apply(lambda x: x.replace('MEL', 'melanoma'))

#concat both 2019 and 2020 data
df_train = pd.concat([df_train, df_train2]).reset_index(drop=True)

# shuffle data
df = df_train.sample(frac=1).reset_index(drop=True)

# creating 8 different target values
new_target = {d: idx for idx, d in enumerate(sorted(df.diagnosis.unique()))}
df['target'] = df['diagnosis'].map(new_target)
mel_idx = new_target['melanoma']

df = df[['filepath','diagnosis', 'target', 'is_ext']]

class_names = list(df['diagnosis'].unique())

# splitting train and validation data by 20%
df_valid = df[:11471]
df_train = df[11472:].reset_index()
df_train = df_train.drop(columns = ['index'])

# create model

def create_model(n_classes):
    model = models.resnet50(pretrained=True)

    n_features = model.fc.in_features
    model.fc = nn.Linear(n_features, n_classes)
    return model.to(device)

# model
base_model = create_model(len(class_names))

# train model
base_model, history = train(base_model, device, num_epochs = 15)

Code Objective
The purpose of the project is to classify skin cancer images by creating 8 different target variables from the given datasets (i.e the competition was about classifying benign and malignant images but I used the diagnosis column on the dataset as my target variable as the data was really skewed).  The model used is Resnet-50 from torchvision models.
These were the data used
skin images (this year competition): https://www.kaggle.com/cdeotte/jpeg-melanoma-384x384
skin images (last year competition): https://www.kaggle.com/cdeotte/jpeg-isic2019-384x384
I decided to create a Flask application out of this but, the CUDA memory was always causing a runtime error
RuntimeError: CUDA out of memory. Tried to allocate 144.00 MiB (GPU 0; 2.00 GiB total capacity; 1.21 GiB already allocated; 43.55 MiB free; 1.23 GiB reserved in total by PyTorch)

These are the details about my Nvidia GPU
Sun Sep 13 19:09:34 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 451.67       Driver Version: 451.67       CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce 940MX      WDDM  | 00000000:01:00.0 Off |                  N/A |
| N/A   63C    P8    N/A /  N/A |     37MiB /  2048MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

# more information about my GPU
==============NVSMI LOG==============

Timestamp                                 : Sun Sep 13 19:11:22 2020
Driver Version                            : 451.67
CUDA Version                              : 11.0

Attached GPUs                             : 1
GPU 00000000:01:00.0
Product Name                          : GeForce 940MX
Product Brand                         : GeForce
Display Mode                          : Disabled
Display Active                        : Disabled
Persistence Mode                      : N/A
Accounting Mode                       : Disabled
Accounting Mode Buffer Size           : 4000
Driver Model
    Current                           : WDDM
    Pending                           : WDDM
Serial Number                         : N/A
GPU UUID                              : GPU-9a8c69df-26f2-2a98-3712-ea22f6add038
Minor Number                          : N/A
VBIOS Version                         : 82.08.6D.00.8C
MultiGPU Board                        : No
Board ID                              : 0x100
GPU Part Number                       : N/A
Inforom Version
    Image Version                     : N/A
    OEM Object                        : N/A
    ECC Object                        : N/A
    Power Management Object           : N/A
GPU Operation Mode
    Current                           : N/A
    Pending                           : N/A
GPU Virtualization Mode
    Virtualization Mode               : None
    Host VGPU Mode                    : N/A
IBMNPU
    Relaxed Ordering Mode             : N/A
PCI
    Bus                               : 0x01
    Device                            : 0x00
    Domain                            : 0x0000
    Device Id                         : 0x134D10DE
    Bus Id                            : 00000000:01:00.0
    Sub System Id                     : 0x83F9103C
    GPU Link Info
        PCIe Generation
            Max                       : 3
            Current                   : 1
        Link Width
            Max                       : 4x
            Current                   : 4x
    Bridge Chip
        Type                          : N/A
        Firmware                      : N/A
    Replays Since Reset               : 0
    Replay Number Rollovers           : 0
    Tx Throughput                     : 0 KB/s
    Rx Throughput                     : 0 KB/s
Fan Speed                             : N/A
Performance State                     : P8
Clocks Throttle Reasons
    Idle                              : Not Active
    Applications Clocks Setting       : Not Active
    SW Power Cap                      : Not Active
    HW Slowdown                       : Not Active
        HW Thermal Slowdown           : N/A
        HW Power Brake Slowdown       : N/A
    Sync Boost                        : Not Active
    SW Thermal Slowdown               : Not Active
    Display Clock Setting             : Not Active
FB Memory Usage
    Total                             : 2048 MiB
    Used                              : 37 MiB
    Free                              : 2011 MiB
BAR1 Memory Usage
    Total                             : 256 MiB
    Used                              : 225 MiB
    Free                              : 31 MiB
Compute Mode                          : Default
Utilization
    Gpu                               : 0 %
    Memory                            : 0 %
    Encoder                           : N/A
    Decoder                           : N/A
Encoder Stats
    Active Sessions                   : 0
    Average FPS                       : 0
    Average Latency                   : 0
FBC Stats
    Active Sessions                   : 0
    Average FPS                       : 0
    Average Latency                   : 0
Ecc Mode
    Current                           : N/A
    Pending                           : N/A
ECC Errors
    Volatile
        Single Bit
            Device Memory             : N/A
            Register File             : N/A
            L1 Cache                  : N/A
            L2 Cache                  : N/A
            Texture Memory            : N/A
            Texture Shared            : N/A
            CBU                       : N/A
            Total                     : N/A
        Double Bit
            Device Memory             : N/A
            Register File             : N/A
            L1 Cache                  : N/A
            L2 Cache                  : N/A
            Texture Memory            : N/A
            Texture Shared            : N/A
            CBU                       : N/A
            Total                     : N/A
    Aggregate
        Single Bit
            Device Memory             : N/A
            Register File             : N/A
            L1 Cache                  : N/A
            L2 Cache                  : N/A
            Texture Memory            : N/A
            Texture Shared            : N/A
            CBU                       : N/A
            Total                     : N/A
        Double Bit
            Device Memory             : N/A
            Register File             : N/A
            L1 Cache                  : N/A
            L2 Cache                  : N/A
            Texture Memory            : N/A
            Texture Shared            : N/A
            CBU                       : N/A
            Total                     : N/A
Retired Pages
    Single Bit ECC                    : N/A
    Double Bit ECC                    : N/A
    Pending Page Blacklist            : N/A
Remapped Rows                         : N/A
Temperature
    GPU Current Temp                  : 60 C
    GPU Shutdown Temp                 : 99 C
    GPU Slowdown Temp                 : 94 C
    GPU Max Operating Temp            : 90 C
    Memory Current Temp               : N/A
    Memory Max Operating Temp         : N/A
Power Readings
    Power Management                  : N/A
    Power Draw                        : N/A
    Power Limit                       : N/A
    Default Power Limit               : N/A
    Enforced Power Limit              : N/A
    Min Power Limit                   : N/A
    Max Power Limit                   : N/A
Clocks
    Graphics                          : 405 MHz
    SM                                : 405 MHz
    Memory                            : 405 MHz
    Video                             : 396 MHz
Applications Clocks
    Graphics                          : 1006 MHz
    Memory                            : 1001 MHz
Default Applications Clocks
    Graphics                          : 1004 MHz
    Memory                            : 1001 MHz
Max Clocks
    Graphics                          : 1241 MHz
    SM                                : 1241 MHz
    Memory                            : 1001 MHz
    Video                             : 1216 MHz
Max Customer Boost Clocks
    Graphics                          : N/A
Clock Policy
    Auto Boost                        : N/A
    Auto Boost Default                : N/A
Processes                             : None

if I try running this on the CPU, the whole system freezes to the point where I have to manually restart the computer. Also if I try running the code with lower image resolution, lower batch sizes etc, each epoch takes around 12 hours to complete on a CPU which is definitely impractical.

Output:
I ran your model on Kaggle with a batch_size = 48 and attached a screenshot of the requirements. An epoch takes around 30-40 mins to complete. I would say you could easily train your model with the 30+ hrs Kaggle gives.

I also tested inference with batch_size=1 and set num_workers=0 in your dataloader, the GPU Usage is 1.3GB.

I would recommend you to train your model on Kaggle/Colab and download the weights onto your local machine. Later, you could run inference on your machine with  batch size = 1. Inference, usually happens faster.


Instruction: How to convert a torch tensor into a byte string?

Input:
I'm trying to serialize a torch tensor using protobuf and it seems using BytesIO along with torch.save() doesn't work. I have tried:
import torch
import io
x = torch.randn(size=(1,20))
buff = io.BytesIO()
torch.save(x, buff)
print(f'buffer: {buff.read()}')

to no avail as it results in b'' in the output! How should I be going about this?

Output:
You need to seek to the beginning of the buffer before reading:
import torch
import io
x = torch.randn(size=(1,20))
buff = io.BytesIO()
torch.save(x, buff)
buff.seek(0)  # <--  this is what you were missing
print(f'buffer: {buff.read()}')

gives you this magnificent output:

buffer: b'PK\x03\x04\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x12\x00archive/data.pklFB\x0e\x00ZZZZZZZZZZZZZZ\x80\x02ctorch._utils\n_rebuild_tensor_v2\nq\x00((X\x07\x00\x00\x00storageq\x01ctorch\nFloatStorage\nq\x02X\x0f\x00\x00\x00140417054790352q\x03X\x03\x00\x00\x00cpuq\x04K\x14tq\x05QK\x00K\x01K\x14\x86q\x06K\x14K\x01\x86q\x07\x89ccollections\nOrderedDict\nq\x08)Rq\ttq\nRq\x0b.PK\x07\x08\xf3\x08u\x13\xa8\x00\x00\x00\xa8\x00\x00\x00PK\x03\x04\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c\x00\x0e\x00archive/data/140417054790352FB\n\x00ZZZZZZZZZZ\xba\xf3x?\xb5\xe2\xc4=)R\x89\xbfM\x08\x19\xbfo%Y\xbf\x05\xc0_\xbf\x03N4\xbe\xdd_ \xc0&\xc4\xb5?\xa7\xfd\xc4?f\xf1$?Ll\xa6?\xee\x8e\x80\xbf\x88Uq?.<\xd8?{\x08\xb2?\xb3\xa3\xba>q\xcd\xbc?\xba\xe3h\xbd\xcan\x11\xc0PK\x07\x08A\xf3\xdc>P\x00\x00\x00P\x00\x00\x00PK\x03\x04\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0f\x003\x00archive/versionFB/\x00ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ3\nPK\x07\x08\xd1\x9egU\x02\x00\x00\x00\x02\x00\x00\x00PK\x01\x02\x00\x00\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00\xf3\x08u\x13\xa8\x00\x00\x00\xa8\x00\x00\x00\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00archive/data.pklPK\x01\x02\x00\x00\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00A\xf3\xdc>P\x00\x00\x00P\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf8\x00\x00\x00archive/data/140417054790352PK\x01\x02\x00\x00\x00\x00\x08\x08\x00\x00\x00\x00\x00\x00\xd1\x9egU\x02\x00\x00\x00\x02\x00\x00\x00\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x01\x00\x00archive/versionPK\x06\x06,\x00\x00\x00\x00\x00\x00\x00\x1e\x03-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\xc5\x00\x00\x00\x00\x00\x00\x00\x12\x02\x00\x00\x00\x00\x00\x00PK\x06\x07\x00\x00\x00\x00\xd7\x02\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00PK\x05\x06\x00\x00\x00\x00\x03\x00\x03\x00\xc5\x00\x00\x00\x12\x02\x00\x00\x00\x00'


Instruction: Is my batch accumulation implementation correct?

Input:
I’d like to know if my code for training a model with batch accumulation is correct. Especially the part regarding the loss calculation because I’m not so sure this is the right way to do this.
Here’s my code:
def train (start_epochs, n_epochs, best_acc, train_generator, val_generator, model, optimizer, criterion, checkpoint_path, best_model_path):

#num_epochs = 25
  since = time.time()

  #best_model_wts = copy.deepcopy(model.state_dict())
  #best_acc = 0.0
  train_loss = []
  val_loss = []
  train_acc = []
  val_acc = []

  batch_accumulation = 8

  for epoch in tqdm(range(start_epochs, n_epochs+1)):

    running_train_loss = 0.0
    running_val_loss = 0.0

    running_train_corrects = 0
    running_val_corrects = 0

    optimizer.zero_grad
    #Training
    model.train()
    for i, (faces, labels) in tqdm(enumerate(train_generator)):

      faces = faces.to(device)
      labels = labels.to(device)

      #forward
      outputs = model(faces)

      #predictions of the model determined using the torch.max() function, which returns the index of the maximum value in a tensor.
      _, preds = torch.max(outputs[1], 1)

      #pass the model outputs and the true image labels to the loss function
      loss = criterion(outputs[1], labels)
      #loss = loss / batch_accumulation
      running_train_loss += loss.item()
      # Backprop and Adam optimisation
      loss.backward()
      # Track the accuracy and loss
      running_train_corrects += torch.sum(preds == labels.data)

      if (i+1)% batch_accumulation == 0:
        optimizer.step()
        optimizer.zero_grad # zero the gradient buffers

    # calculate average losses and accuracy
    epoch_train_loss = running_train_loss / len(train_generator.dataset)
    epoch_train_acc = ((running_train_corrects.double() / len(train_generator.dataset)) * 100)
    train_loss.append(epoch_train_loss)
    train_acc.append(epoch_train_acc)

    print('Train Loss: {:.4f} Train Acc: {:.2f}%'.format(epoch_train_loss, epoch_train_acc))

    #Validation
    with torch.set_grad_enabled(False):
      model.eval()
      for i , (faces_val, labels_val) in tqdm(enumerate(val_generator)):

        faces_val = faces_val.to(device)
        labels_val = labels_val.to(device)

        if (i+1)% batch_accumulation == 0:

          outputs_val = model(faces_val)

          _, preds_val = torch.max(outputs_val[1], 1)
          loss_val = criterion(outputs_val[1], labels_val)

          running_val_loss += loss_val.item()
          #running_val_loss = running_val_loss +((1 /(i+1)) * (loss.item() - running_val_loss))
          running_val_corrects += torch.sum(preds_val == labels_val.data)

    # calculate average losses and accuracy
    epoch_val_loss = running_val_loss / len(validation_generator.dataset)
    epoch_val_acc = (running_val_corrects.double() / len(validation_generator.dataset)) * 100
    val_loss.append(epoch_val_loss)
    val_acc.append(epoch_val_acc)

    print('Validation Loss: {:.4f} Validation Acc: {:.2f}%'.format(epoch_val_loss, epoch_val_acc))

I got strange epoch train results (like 456.890) and I’m note sure about the if statement in the validation part.

Output:
There is no need for gradient accumulation (actual term) during validation phase, so this part over there:
if (i+1)% batch_accumulation == 0:
    outputs_val = model(faces_val)

does not make any sense (no need for if). This technique is used solely for training to make the gradient estimation more accurate for small batches, hence we should focus on it.
Gradient accumulation
Each time we run backward() calculated gradients are added to the leafs of the graph. Usually, we use mean across whole batch (divide the sum by number of elements in batch). Here, we accumulate the loss, hence we should divide it by the number of accumulation steps, which gives us (actually you had it commented out):
loss = criterion(outputs[1], labels)
loss = loss / batch_accumulation

Otherwise loss might be too big (probably the case here) and make the network unstable even with really small learning rates).
You may also run this:
running_train_loss += loss.item()

On a per-accumulation basis.
And lastly, as pointed out by @Dishin H Goyani zero_grad is a function so you should run:
optimizer.zero_grad()


Instruction: Neural network layer without all connections

Input:
The weights in a dense layer of a neural network is a (n,d) matrix, and I want to force some of these weights to always be zero.  I have another (n,d) matrix which is the mask of which entries can be non-zero.  The idea is that the layer should not be truly dense, but have some connections missing (i.e. equal to 0).
How can achieve this while training with PyTorch (or Tensorflow)?  I don't want these weights to become non-zero while training.
One method, if it doesn't support it directly, would be to zero-out the desired entries after each iteration of training.

Output:
You can take advantage of pytorch's sparse data type:
class SparseLinear(nn.Module):
  def __init__(self, in_features, out_features, sparse_indices):
    super(SparseLinear, self).__init__()
    self.weight = nn.Parameter(data=torch.sparse.FloatTensor(sparse_indices, torch.randn(sparse_indices.shape[1]), [in_features, out_features]), requires_grad=True)
    self.bias = nn.Parameter(data=torch.randn(out_features), requires_grad=True)

  def forward(self, x):
    return torch.sparse.admm(self.bias, self.weight, x, 1., 1.)


Instruction: Pytorch NLP model doesn’t use GPU when making inference

Input:
I have a NLP model trained on Pytorch to be run in Jetson Xavier. I installed Jetson stats to monitor usage of CPU and GPU. When I run the Python script, only CPU cores work on-load, GPU bar does not increase. I have searched on Google about that with keywords of " How to check if pytorch is using the GPU?" and checked results on stackoverflow.com etc. According to their advices to someone else facing similar issue, cuda is available and there is cuda device in my Jetson Xavier. However, I don’t understand why GPU bar does not change, CPU core bars go to the ends.
I don’t want to use CPU, it takes so long to compute. In my opinion, it uses CPU, not GPU. How can I be sure and if it uses CPU, how can I change it to GPU?
Note: Model is taken from huggingface transformers library. I have tried to use cuda() method on the model. (model.cuda()) In this scenario, GPU is used but I can not get an output from model and raises exception.
Here is the code:
from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline
import torch

BERT_DIR = "savasy/bert-base-turkish-squad"

tokenizer = AutoTokenizer.from_pretrained(BERT_DIR)
model = AutoModelForQuestionAnswering.from_pretrained(BERT_DIR)
nlp=pipeline("question-answering", model=model, tokenizer=tokenizer)

def infer(question,corpus):
    try:
        ans = nlp(question=question, context=corpus)
        return ans["answer"], ans["score"]
    except:
        ans = None
        pass

    return None, 0

Output:
The problem has been solved with loading pipeline containing device parameter:
nlp = pipeline("question-answering", model=BERT_DIR, device=0)


Instruction: Using Transformer for Text-Summarization

Input:
I am using huggingface transformer models for text-summarization.
Currently I am testing different models such as T5 and Pegasus.
Now these models were trained for summarizing Big Texts into very short like a maximum of two sentences. Now I have the task, that I want summarizations, that are about half the size of the text, ergo the generated summaries are too small for my purpose.
My question now is, if there is a way to tell the model that another sentence came before?
Kind of similar to the logic inside stateful RNNs (although I know they work completly different).
If yes, I could summarize small windows over the sentences always with the information which content came before.
Is that just a thing of my mind? I cant believe that I am the only one, who wants to create shorter summaries, but not only 1 or two sentence long ones.
Thank you

Output:
Why not transfer learning? Train them on your specific texts and summaries.
I trained T5 on specific limited text over 5 epoch and got very good results. I adopted the code from here to my needs https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb
Let me know if you have a specific training questions.


Instruction: Specifying number of cells in LSTM layer in PyTorch

Input: I don't fully understand the LSTM layer in PyTorch. When I instantiate an LSTM layer how can I specify the number of LSTM cells inside the layer? My first thought was that it was the "num_layers" argument, if we assume that LSTM cells are connected vertically. But if that is the case how can we implement stacked LSTM with for example two layers with 8 cells each?

Output:
The amount of cells of an LSTM (or RNN or GRU) is the amount of timesteps your input has/needs. For example, when you want to run the word „hello“ through the LSTM function in Pytorch, you can just convert the word to a vector (with one-hot encoding or embeddings) and then pass that vector though the LSTM function. It will then, in the background, iterate through all the embedded characters („h“, „e“, „l“, ...). And each input can even have a different amount of timesteps/cells, for example when you want to pass „hello“ and after that „Joe“ the LSTM will need different amount of iterations (5 for hello, 3 for Joe). So as you can see, there is no need to give an amount of cells! Hope that answer satisfied you. :)
Edit
An example:
sentence = "Hey Im Joe"

embedding_size = 300
batch_size = 1  # batch-size 1 for demonstration

input_ = [create_embedding(word, dims=embedding_size) for word in sentence]
# the LSTM will need three timesteps or cells to process that sentence
input_ = torch.tensor(x).reshape(1, embedding_size)

hidden_size = 256
layers = 256

lstm = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size, num_layers=layers, dropout=0.5, batch_first=True)

# initialize hidden-state (must be tuple of following dimentsions
hidden = (torch.zeros(layers, batch_size, hidden_size), torch.zeros(layers, batch_size, hidden_size))

outputs, hidden = lstm(input_, hidden)
# outputs is now a list containing the outputs of each timestep
# for classification you can take output of the last timestep and use it further, like this
output = outputs[:, -1]

So what happens in this (outputs, hidden = lstm(input, hidden)) line?
Again pseudo code:
# inside the LSTM function

# lets say for demonstration reasons the embedding for "Hey" is a, for "Im" its b and for "Joe" its c

input_ = [a, b, c]

def LSTM(input_sentence):
    hidden = ...
    outputs = []

    # each iteration is one timestep/cell
    for embedded_word in input_sentence:
        output, hidden = neural_network(embedded_word, hidden)
        outputs.append(output)

    # returns all outputs and hiddenstate (which you normally dont need)
    return outputs, hidden

outputs, hidden = LSTM(input_)

Is it clear now how what the LSTM function does and how to use it?


Instruction: Why there are orders of magnitude in the evaluation of torch.sin() and numpy.sin() at integer multiples of Pi?

Input:
Why are trigonometric functions of Pytorch and Numpy when evaluated at integer multiples of Pi result in such huge discrepancy in the orders of magnitude?
>>> torch.sin(torch.ones(1)*2*np.pi)
tensor([1.7485e-07])
>>> np.sin(np.ones(1)*2*np.pi)
array([-2.4492936e-16])

Output: Torch defaults to 32-bit floats, while NumPy defaults to 64-bit. The rounding error you're getting in Torch is around the expected scale for 32-bit floats. Specify a different dtype if you want a different dtype.


Instruction: Randomly select items from two equally sized tensors

Input:
Assume that we have two equally sized tensors of size batch_size * 1. For each index in the batch dimension we want to choose randomly between the two tensors. My solution was to create an indices tensor that contains random 0 or 1 indices of size batch_size and use those to index_select from the concatenation of the two tensors. However, to do so I had the "view" that cat tensor and the solution ended up to be quite "ugly":
import torch

bs = 8
a = torch.zeros(bs, 1)
print("a size", a.size())
b = torch.ones(bs, 1)

c = torch.cat([a, b], dim=-1)
print(c)
print("c size", c.size())

# create bs number of random 0 and 1's
indices = torch.randint(0, 2, [bs])
print("idxs size", indices.size())
print("idxs", indices)

# use `indices` to slice the `cat`ted tensor
d = c.view(1, -1).index_select(-1, indices).view(-1, 1)
print("d size", d.size())
print(d)

I am wondering whether there is a prettier and, more importantly, more efficient solution.

Output:
Posting two answers that I got over at the PyTorch forums
import torch

bs = 8
a = torch.zeros(bs, 1)
b = torch.ones(bs, 1)
c = torch.cat([a, b], dim=-1)
choices_flat = c.view(-1)
# index = torch.randint(choices_flat.numel(), (bs,))
# or if replace = False
index = torch.randperm(choices_flat.numel())[:bs]
select = choices_flat[index]

print(select)

import torch

bs = 8
a = torch.zeros(bs, 1)
print("a size", a.size())
b = torch.ones(bs, 1)

idx = torch.randint(2 * bs, (bs,))

d = torch.cat([a, b])[idx] # [bs, 1]


Instruction: Where is the definition of torch.embedding in PyTorch?

Input:
I'm trying to understand how PyTorch creates embeddings and read the source code of torch.nn.functional.embedding github link.
The function returns the result of torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse).
Then, I try to understand the definition of torch.embedding but I can't find its source code in the GitHub repository. Where is it?

Output: Many of PyTorch's functions are implemented in C++. The entrypoint for torch.embedding is located here.


Instruction: Is that normal to train properly with 2 sets and/or 2 diff losses function?

Input:
I have 2 train sets: one with label and one with no label.
When training, i’m simultaneously loading one batch from a labelled set, then calculating using the first loss function; and then, one batch from unlabeled set, calculating using the using the other function. Finally I sum them (2 losses) and loss.backward() .
Does this way work ? it’s quite uncommon in my mind so just ask if the engine know how to back-propagate properly (not wrong)?
Thank you.

Output:
I've got an answer from pytorch forum discussion. Autograd engine in pytorch work well and correctly with 2 or more different loss function so we don't need to worry about its accuracy.
Thanks


Instruction: Get value from c10::Dict in Pytorch C++

Input:
I'm using a TorchScript Model on Pytorch C++ Frontend.
The model in Python returns an output dict as  Dict[str, List[torch.Tensor]].
When I use it in C++, it returns a c10::Dict<c10::IValue, c10::IValue>. What is the equivalent of this Python code:
value_a = output['key_a']
value_b = output['key_b']

in C++ to get value from c10::Dict?
I have tried this but it does not work.
torch::IValue key_a("key_a");
torch::IValue key_b("key_b");
c10::IValue value_a = output[key_a];
c10::IValue value_b = output[key_b];
std::cout << value_a << std::endl;
std::cout << value_b << std::endl;

And error:
error: type 'c10::Dict<c10::IValue, c10::IValue>' does not provide a subscript operator

Output:
You can find header file of c10:Dict here. What you want is at method (defined here), so:
auto value_a = output.at(key_a);

Should do the trick.
Also you don't have to create torch::IValue key_ay("key_a") explicitly, this should be sufficient:
auto value_a = output.at("key_a");


Instruction: Unittest the pytorch forward function

Input:
I want to unittest the overridden forward function of my Network modell in Pytorch. So I loaded my model (pretrained from Zoo) with the setUp method, loaded a seed and created some random batch. In my method testForward I tested the result of forward against shape and numel, but I also want to check a specific value which a apears to be 0. I wasn't shure about that so checked my params in setUp also, which appears not to be 0.
import unittest
import torch
from SemanticSegmentation.models.fcn8 import FCN8

class TestFCN8(unittest.TestCase):

    def setUp(self):
        self.model = FCN8(8, pretrained=True)
        torch.manual_seed(0)
        self.x = torch.rand((4, 3, 45, 45))
        for param in self.model.parameters():
            print(param.data)

    def testForward(self):
        self.assertEqual(self.model.forward(self.x).shape.numel(), 64800)
        self.assertEqual(str(self.model.forward(self.x).shape), 'torch.Size([4, 8, 45, 45])')
        print(self.model.named_parameters)

if __name__ == "__main__":
    unittest.main()

So my Question is: the sahpe of the forward return tensor is what I expect but why is this tensor completly zero? I expected at least a few values.
The imported modell is based on an VGG16 network with upscoring the after ConvLayer 4 , 8 and 16. If needed I could also present the modell code.

Output:
Ok after tinkering around and debugging the forward function I came to following explanation:
Some Information about the architecture
If you do classes from Andrew Ng or others you learn not to initialize the weights to the same value, as example "0". This is what the writers of the original Paper of FCNs do and they say, because it doesn't change the performance or didn't yield to faster convergence (FCN-Paper).
My Solution
So for testing purpose I initlize in the testing module to seeded random values, which I can test against:
import unittest
import torch
from SemanticSegmentation.models.fcn8 import FCN8

class TestFCN8(unittest.TestCase):

    def setUp(self):
        self.model = FCN8(8, pretrained=True)
        torch.manual_seed(0)
        # instead of zero init for score tensors use random init
        self.model.score_fr[6].weight.data.random_()
        self.model.score_fr[6].bias.data.random_()
        self.model.score_pool3.weight.data.random_()
        self.model.score_pool3.bias.data.random_()
        self.model.score_pool4.weight.data.random_()
        self.model.score_pool4.bias.data.random_()
        self.x = torch.rand((4, 3, 45, 45))

    def testForward(self):
        self.assertEqual(
            self.model.forward(self.x).shape.numel(), 64800)
        self.assertEqual(
            list(self.model.forward(self.x).shape), [4, 8, 45, 45])
        self.assertEqual(
            float(self.model.forward(self.x)[3][4][44][4]), 2277257216.0))

if __name__ == "__main__":
    unittest.main()


Instruction: Whitelist tokens for text generation (XLNet, GPT-2) in huggingface-transformers

Input:
In the documentation on text generation (https://huggingface.co/transformers/main_classes/model.html#generative-models) there is the option to put
bad_words_ids (List[int], optional) – List of token ids that are not allowed to be generated. In order to get the tokens of the words that should not appear in the generated text, use tokenizer.encode(bad_word, add_prefix_space=True).

Is there also the option to put something along the lines of "allowed_words_ids"? The idea would be to restrict the language of the generated texts.

Output:
I'd also suggest to do what Sahar Mills said. You can do it in the following way.

You get the whole vocab of the model you are using, e.g.

from transformers import AutoTokenizer

# Load tokenizer
checkpoint = "CenIA/distillbert-base-spanish-uncased" #Example model
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

vocab = tokenizer.get_vocab()
list(vocab.keys())[:100] # to see the first 100 words

Define words you do want in the model.

words_to_delete = ['forzado', 'vendieron', 'verticales'] # or load them from somewhere else

Define function to create the bad_words_ids, that is, the whole model vocab minus the words you want in the model

def create_bad_words_ids(bad_words_ids, words_to_delete):
  for pictogram in range(len(words_to_delete)):
    if words_to_delete[pictogram] in bad_words_ids:
      bad_words_ids.remove(words_to_delete[pictogram])
  return bad_words_ids

bad_words_ids = create_bad_words_ids(bad_words_ids=bad_words_ids, words_to_delete=words_to_delete)
print(bad_words_ids)

Hope it helps,
cheers


Instruction: Intercepting CUDA calls

Input: I am trying to intercept cudaMemcpy calls from the pytorch library for analysis. I noticed NVIDIA has a cuHook example in the CUDA toolkit samples. However that example requires one to modify the source code of the application itself which I cannot do in this case. So is there a way to write a hook to intercept CUDA calls without modifying the application source code?

Output:
A CUDA runtime API call can be hooked (on linux) using the "LD_PRELOAD trick" if the application that is being run is dynamically linked to the CUDA runtime library (libcudart.so).
Here is a simple example on linux:
$ cat mylib.cpp
#include <stdio.h>
#include <unistd.h>
#include <dlfcn.h>
#include <cuda_runtime.h>

cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind )
{
cudaError_t (*lcudaMemcpy) ( void*, const void*, size_t, cudaMemcpyKind) = (cudaError_t (*) ( void* , const void* , size_t , cudaMemcpyKind  ))dlsym(RTLD_NEXT, "cudaMemcpy");
    printf("cudaMemcpy hooked\n");
    return lcudaMemcpy( dst, src, count, kind );
}

cudaError_t cudaMemcpyAsync ( void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t str )
{
cudaError_t (*lcudaMemcpyAsync) ( void*, const void*, size_t, cudaMemcpyKind, cudaStream_t) = (cudaError_t (*) ( void* , const void* , size_t , cudaMemcpyKind, cudaStream_t   ))dlsym(RTLD_NEXT, "cudaMemcpyAsync");
    printf("cudaMemcpyAsync hooked\n");
    return lcudaMemcpyAsync( dst, src, count, kind, str );
}
$ g++ -I/usr/local/cuda/include -fPIC -shared -o libmylib.so mylib.cpp -ldl -L/usr/local/cuda/lib64 -lcudart
$ cat t1.cu
#include <stdio.h>

int main(){

  int a, *d_a;
  cudaMalloc(&d_a, sizeof(d_a[0]));
  cudaMemcpy(d_a, &a, sizeof(a), cudaMemcpyHostToDevice);
  cudaStream_t str;
  cudaStreamCreate(&str);
  cudaMemcpyAsync(d_a, &a, sizeof(a), cudaMemcpyHostToDevice);
  cudaMemcpyAsync(d_a, &a, sizeof(a), cudaMemcpyHostToDevice, str);
  cudaDeviceSynchronize();
}
$ nvcc -o t1 t1.cu -cudart shared
$ LD_LIBRARY_PATH=/usr/local/cuda/lib64 LD_PRELOAD=./libmylib.so cuda-memcheck ./t1
========= CUDA-MEMCHECK
cudaMemcpy hooked
cudaMemcpyAsync hooked
cudaMemcpyAsync hooked
========= ERROR SUMMARY: 0 errors
$

(CentOS 7, CUDA 10.2)
A simple test with pytorch seems to indicate that it works:
$ docker run --gpus all -it nvcr.io/nvidia/pytorch:20.08-py3
...
Status: Downloaded newer image for nvcr.io/nvidia/pytorch:20.08-py3

=============
== PyTorch ==
=============

NVIDIA Release 20.08 (build 15516749)
PyTorch Version 1.7.0a0+8deb4fe

Container image Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.

Copyright (c) 2014-2020 Facebook Inc.
Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
Copyright (c) 2011-2013 NYU                      (Clement Farabet)
Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
Copyright (c) 2015      Google Inc.
Copyright (c) 2015      Yangqing Jia
Copyright (c) 2013-2016 The Caffe contributors
All rights reserved.

Various files include modifications (c) NVIDIA CORPORATION.  All rights reserved.
NVIDIA modifications are covered by the license terms that apply to the underlying project or file.

NOTE: MOFED driver for multi-node communication was not detected.
      Multi-node communication performance may be reduced.

NOTE: The SHMEM allocation limit is set to the default of 64MB.  This may be
   insufficient for PyTorch.  NVIDIA recommends the use of the following flags:
   nvidia-docker run --ipc=host ...

...
root@946934df529b:/workspace# cat mylib.cpp
#include <stdio.h>
#include <unistd.h>
#include <dlfcn.h>
#include <cuda_runtime.h>

cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind )
{
cudaError_t (*lcudaMemcpy) ( void*, const void*, size_t, cudaMemcpyKind) = (cudaError_t (*) ( void* , const void* , size_t , cudaMemcpyKind  ))dlsym(RTLD_NEXT, "cudaMemcpy");
    printf("cudaMemcpy hooked\n");
    return lcudaMemcpy( dst, src, count, kind );
}

cudaError_t cudaMemcpyAsync ( void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t str )
{
cudaError_t (*lcudaMemcpyAsync) ( void*, const void*, size_t, cudaMemcpyKind, cudaStream_t) = (cudaError_t (*) ( void* , const void* , size_t , cudaMemcpyKind, cudaStream_t   ))dlsym(RTLD_NEXT, "cudaMemcpyAsync");
    printf("cudaMemcpyAsync hooked\n");
    return lcudaMemcpyAsync( dst, src, count, kind, str );
}
root@946934df529b:/workspace# g++ -I/usr/local/cuda/include -fPIC -shared -o libmylib.so mylib.cpp -ldl -L/usr/local/cuda/lib64 -lcudart
root@946934df529b:/workspace# cat tt.py
import torch
device = torch.cuda.current_device()
x = torch.randn(1024, 1024).to(device)
y = torch.randn(1024, 1024).to(device)
z = torch.matmul(x, y)
root@946934df529b:/workspace# LD_LIBRARY_PATH=/usr/local/cuda/lib64 LD_PRELOAD=./libmylib.so python tt.py
cudaMemcpyAsync hooked
cudaMemcpyAsync hooked
root@946934df529b:/workspace#

(using NVIDIA NGC PyTorch container )


Instruction: GPT2 on Hugging face(pytorch transformers) RuntimeError: grad can be implicitly created only for scalar outputs

Input:
I am trying to fine-tune gpt2 with a custom dataset of mine. I created a basic example with the documentation from hugging-face transformers. I receive the mentioned error. I know what it means: (basically it is calling backward on a non-scalar tensor) but since I almost use only API calls, I have no idea how to fix this issue. Any suggestions?
from pathlib import Path
from absl import flags, app
import IPython
import torch
from transformers import GPT2LMHeadModel, Trainer,  TrainingArguments
from data_reader import GetDataAsPython

# this is my custom data, but i get the same error for the basic case below
# data = GetDataAsPython('data.json')
# data = [data_point.GetText2Text() for data_point in data]
# print("Number of data samples is", len(data))

data = ["this is a trial text", "this is another trial text"]

train_texts = data

from transformers import GPT2Tokenizer
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')

special_tokens_dict = {'pad_token': '<PAD>'}
num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)
train_encodigs = tokenizer(train_texts, truncation=True, padding=True)

class BugFixDataset(torch.utils.data.Dataset):
    def __init__(self, encodings):
        self.encodings = encodings

    def __getitem__(self, index):
        item = {key: torch.tensor(val[index]) for key, val in self.encodings.items()}
        return item

    def __len__(self):
        return len(self.encodings['input_ids'])

train_dataset = BugFixDataset(train_encodigs)

training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=3,
    per_device_train_batch_size=1,
    per_device_eval_batch_size=1,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir='./logs',
    logging_steps=10,
)

model = GPT2LMHeadModel.from_pretrained('gpt2', return_dict=True)
model.resize_token_embeddings(len(tokenizer))

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
)

trainer.train()

Output:
I finally figured it out. The problem was that the data samples did not contain a target output. Even tough gpt is self-supervised, this has to be explicitly told to the model.
you have to add the line:
item['labels'] = torch.tensor(self.encodings['input_ids'][index])

to the getitem function of the Dataset class and then it runs okay!


Instruction: Can't iterate through PyTorch DataLoader

Input:
I am trying to learn PyTorch and create my first neural network. I am using a custom dataset, here is a sample of the data:
ID_REF  cg00001854  cg00270460  cg00293191  cg00585219  cg00702638  cg01434611  cg02370734  cg02644867  cg02879967  cg03036557  cg03123104  cg03670302  cg04146801  cg04570540  cg04880546  cg07044749  cg07135408  cg07303143  cg07475178  cg07553761  cg07917901  cg08016257  cg08548498  cg08715791  cg09334636  cg11153071  cg11441796  cg11642652  cg12256803  cg12352902  cg12541127  cg13313833  cg13500819  cg13975075  cg14061946  cg14086922  cg14224196  cg14530143  cg15456742  cg16230982  cg16734549  cg17166941  cg17290213  cg17292667  cg18266594  cg18335535  cg18584803  cg19273773  cg19378199  cg19523692  cg20115827  cg20558024  cg20608895  cg20899581  cg21186299  cg22115892  cg22454769  cg22549547  cg23098693  cg23193759  cg23500537  cg23606718  cg24079702  cg24888989  cg25090514  cg25344401  cg25635000  cg25726357  cg25743481  cg26019498  cg26647566  cg26792755  cg26928195  cg26940620  Age
0   0.252486    0.284724    0.243242    0.200685    0.904132    0.102795    0.473919    0.264084    0.367480    0.671434    0.075955    0.329343    0.217375    0.210861    1.000000    0.356048    0.577945    0.557148    0.249014    0.847134    0.254539    0.319858    0.220589    0.796789    0.361994    0.296101    0.105965    0.239796    0.169738    0.357586    0.365674    0.132575    0.250932    0.283227    1.000000    0.262259    0.208146    0.290623    0.113049    0.255710    0.555382    0.281046    0.168826    0.492007    0.442871    0.509569    0.219183    0.641244    0.339088    0.164062    0.227678    0.340220    0.541491    0.423010    0.621303    0.243750    0.869947    0.124120    0.317660    0.985243    0.645869    0.590888    0.841485    0.825372    0.904037    0.407343    0.223722    0.352113    0.855653    0.289593    0.428849    0.719758    0.800240    0.473586    68
1   0.867671    0.606590    0.803673    0.845942    0.086222    0.996915    0.871998    0.791823    0.877639    0.095326    0.857108    0.959701    0.688322    0.650640    0.062329    0.920434    0.687537    0.193038    0.891809    0.273775    0.583457    0.793486    0.798427    0.102910    0.773496    0.658568    0.759050    0.754877    0.787817    0.585895    0.792240    0.734543    0.854528    0.735642    0.389495    0.736709    0.600386    0.775989    0.819579    0.696350    0.110374    0.878199    0.659849    0.716714    0.771206    0.870711    0.919629    0.359592    0.677752    0.693433    0.683448    0.792423    0.933971    0.170669    0.249908    0.879879    0.111498    0.623053    0.626821    0.000000    0.157429    0.197567    0.160809    0.183031    0.202754    0.597896    0.826429    0.886736    0.086038    0.844088    0.761793    0.056548    0.270670    0.940083    21
2   0.789439    0.594060    0.857086    0.633195    0.000000    0.953293    0.832107    0.692119    0.641294    0.169303    0.935807    0.674698    0.789146    0.796555    0.208590    0.791318    0.777537    0.221895    0.804405    0.138006    0.738616    0.758083    0.749127    0.180998    0.769312    0.592938    0.578885    0.896125    0.553588    0.781393    0.898768    0.705339    0.861029    0.966552    0.274496    0.575738    0.490313    0.951172    0.833724    0.901890    0.115235    0.651489    0.619196    0.760758    0.902768    0.835082    0.610065    0.294962    0.907979    0.703284    0.775867    0.910324    0.858090    0.190595    0.041909    0.792941    0.146005    0.615639    0.761822    0.254161    0.101765    0.343289    0.356166    0.088915    0.114347    0.628616    0.697758    0.910687    0.133282    0.775737    0.809420    0.129848    0.126485    0.875580    20
3   0.615803    0.710968    0.874037    0.771136    0.199428    0.861378    0.861346    0.695713    0.638599    0.158479    0.903668    0.758718    0.581146    0.857357    0.307756    0.977337    0.805049    0.188333    0.788991    0.312119    0.706578    0.782006    0.793232    0.288111    0.691131    0.758102    0.829221    1.000000    0.742666    0.897607    0.797869    0.803221    0.912101    0.736800    0.315636    0.760577    0.609101    0.733923    0.578598    0.796944    0.096960    0.924135    0.612601    0.727117    0.905177    0.776481    0.727865    0.429820    0.666803    0.924595    0.567474    0.752196    0.742709    0.303662    0.168286    0.720899    0.099313    0.595328    0.734024    0.268583    0.293437    0.244840    0.311726    0.213415    0.418673    0.819981    0.816660    0.684730    0.146797    0.686270    0.777680    0.087826    0.335125    1.000000    23
4   0.847329    0.735766    0.858018    0.896453    0.186994    0.831964    0.762522    0.840186    0.830930    0.199264    0.788487    0.912629    0.702284    0.838771    0.065271    0.959230    0.912387    0.377203    0.794480    0.207909    0.766246    0.582117    0.902944    0.301144    0.765401    0.715115    0.646735    0.812084    0.697886    0.714310    0.890658    0.826644    0.944022    0.729517    0.530379    0.756268    0.764899    0.914573    0.825766    0.673394    0.017316    0.949335    0.614375    0.650553    0.898788    0.685396    0.823348    0.210175    0.831852    0.829067    0.858212    0.916433    0.778864    0.241186    0.144072    0.889536    0.058360    0.703567    0.852496    0.094223    0.341236    0.284903    0.231957    0.125196    0.333207    0.752592    0.899356    0.839006    0.174601    0.937948    0.716135    0.000000    0.114062    0.969760    22

I split the data into train/test/val data like this:
train_df, rest_df = train_test_split(df, test_size=0.4)
test_df, val_df = train_test_split(rest_df, test_size=0.5)

x_train_tensor = torch.tensor(train_df.drop('Age', axis=1).to_numpy(), requires_grad=True)
y_train_tensor = torch.tensor(train_df['Age'].to_numpy())

x_test_tensor = torch.tensor(test_df.drop('Age', axis=1).to_numpy(), requires_grad=True)
y_test_tensor = torch.tensor(test_df['Age'].to_numpy())

x_val_tensor = torch.tensor(val_df.drop('Age', axis=1).to_numpy(), requires_grad=True)
y_val_tensor = torch.tensor(val_df['Age'].to_numpy())

bs = len(train_df.index)//10
train_dl = DataLoader(train_df, bs, shuffle=True)
test_dl = DataLoader(test_df, len(test_df), shuffle=False)
val_dl = DataLoader(val_df, bs, shuffle=False)

And here is the Network so far (very basic, just to test if it works):
class Net(nn.Module):
    def __init__(self):
        super().__init__()
        input_size = len(df.columns)-1
        self.fc1 = nn.Linear(input_size, input_size//2)
        self.fc2 = nn.Linear(input_size//2, input_size//4)
        self.fc3 = nn.Linear(input_size//4, 1)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))

        return x

net = Net()
print(net)

Here is where I get the error, on the last line:
loss = torch.nn.MSELoss()
optimizer = optim.Adam(net.parameters(), lr=0.001)

EPOCHS = 3
STEPS_PER_EPOCH = len(train_dl.dataset)//bs
iterator = iter(train_dl)
print(train_dl.dataset)
for epoch in range(EPOCHS):
    for s in range(STEPS_PER_EPOCH):
        print(iterator)
        iterator.next()

ID_REF  cg00001854  cg00270460  cg00293191  ...  cg26928195  cg26940620  Age
29        0.781979    0.744825    0.744579  ...    0.242138    0.854054   19
44        0.185400    0.299145    0.160084  ...    0.638449    0.413286   69
21        0.085470    0.217421    0.277675  ...    0.863455    0.512334   75
4         0.847329    0.735766    0.858018  ...    0.114062    0.969760   22
20        0.457293    0.462984    0.323835  ...    0.584259    0.481060   68
33        0.784562    0.845031    0.958335  ...    0.122210    0.854005   19
25        0.258434    0.354822    0.405620  ...    0.677245    0.540463   70
27        0.737131    0.768188    0.897724  ...    0.203228    0.831175   20
37        0.002051    0.202403    0.134198  ...    0.753844    0.302229   70
10        0.737427    0.537413    0.614343  ...    0.464244    0.723953   23
0         0.252486    0.284724    0.243242  ...    0.800240    0.473586   68
32        0.927260    1.000000    0.853864  ...    0.261990    0.892503   18
7         0.035825    0.271602    0.236109  ...    1.000000    0.471256   69
17        0.000000    0.202986    0.132144  ...    0.874550    0.342981   79
18        0.282112    0.479775    0.218852  ...    0.908217    0.426143   79
11        0.708797    0.536074    0.721171  ...    0.048768    0.699540   27
15        0.686921    0.639198    0.858981  ...    0.305142    0.978350   24
38        0.246031    0.186011    0.235928  ...    0.754013    0.342380   70
30        0.814767    0.771483    0.437789  ...    0.000000    0.658354   18
43        0.247471    0.399231    0.271619  ...    0.895016    0.468336   72
46        0.000428    0.263164    0.163303  ...    0.567005    0.252806   76
3         0.615803    0.710968    0.874037  ...    0.335125    1.000000   23
5         0.777925    0.821814    0.636676  ...    0.233359    0.753266   20
34        0.316262    0.307535    0.203090  ...    0.570755    0.351226   73
23        0.133038    0.000000    0.208442  ...    0.631202    0.459593   76
6         0.746102    0.585211    0.626580  ...    0.311914    0.753994   25
1         0.867671    0.606590    0.803673  ...    0.270670    0.940083   21
47        0.444606    0.502357    0.207560  ...    0.987106    0.446959   71

[28 rows x 75 columns]
<torch.utils.data.dataloader._SingleProcessDataLoaderIter object at 0x7f166241c048>
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2645             try:
-> 2646                 return self._engine.get_loc(key)
   2647             except KeyError:

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 13

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
6 frames
/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2646                 return self._engine.get_loc(key)
   2647             except KeyError:
-> 2648                 return self._engine.get_loc(self._maybe_cast_indexer(key))
   2649         indexer = self.get_indexer([key], method=method, tolerance=tolerance)
   2650         if indexer.ndim > 1 or indexer.size > 1:

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 13

I really have no idea what the error means or where to look.
I'd greatly appreciate some guidance, thank you!

Output:
Use Numpy array instead of dataframe. You can use to_numpy() to convert dataframe to numpy array.
train_dl = DataLoader(train_df.to_numpy(), bs, shuffle=True)
test_dl = DataLoader(test_df.to_numpy(), len(test_df), shuffle=False)
val_dl = DataLoader(val_df.to_numpy(), bs, shuffle=False)


Instruction: Determinant of a complex matrix in PyTorch

Input:
Is there a way to calculate the determinant of a complex matrix in PyTroch?
torch.det is not implemented for 'ComplexFloat'

Output:
Unfortunately it's not implemented currently. One way would be to implement your own version or simply use np.linalg.det.
Here is a short function which computes the determinant of a complex matrix that I wrote using LU-decomposition:
def complex_det(A):
    def complex_diag(A):
        return torch.view_as_complex(torch.stack((A.real.diag(), A.imag.diag()),dim=1))
    #Perform LU decomposition to matrix A:
    A_LU, pivots = A.lu()
    P, A_L, A_U = torch.lu_unpack(A_LU, pivots)
    #Det. of multiplied matrices is multiplcation of det.:
    det = torch.prod(complex_diag(A_L)) * torch.prod(complex_diag(A_U)) * torch.det(P.real) #Could probably calculate det(P) [which is +-1] efficiently using Sylvester's determinant identity
    return det
#Test it:
A = torch.view_as_complex(torch.randn(3,3,2))
complex_det(A)


Instruction: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

Input:
I'm trying to implement ResNet18 on pyTorch but I'm having some troubles with it. My code is this:
device = torch.device("cuda:0")

class ResnetBlock(nn.Module):
    def __init__(self, strides, nf, nf0, reps, bn):
        super(ResnetBlock, self).__init__()
        self.adapt = strides == 2
        self.layers = []
        self.relus = []
        self.adapt_layer = nn.Conv2d(nf0, nf, kernel_size=1, stride=strides, padding=0) if self.adapt else None
        for i in range(reps):
            self.layers.append(nn.Sequential(
                nn.Conv2d(nf0, nf, kernel_size=3, stride=strides, padding=1),
                nn.BatchNorm2d(nf, eps=0.001, momentum=0.99),
                nn.ReLU(),
                nn.Conv2d(nf, nf, kernel_size=3, stride=1, padding=1),
                nn.BatchNorm2d(nf, eps=0.001, momentum=0.99)))
            self.relus.append(nn.ReLU())

            strides = 1
            nf0 = nf

    def forward(self, x):
        for i, (layer, relu) in enumerate(zip(self.layers, self.relus)):
            rama = layer(x)
            if self.adapt and i == 0:
                x = self.adapt_layer(x)
            x = x + rama
            x = relu(x)
        return x

class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3),
            nn.MaxPool2d(kernel_size=2, stride=2))

        self.blocks = nn.Sequential(
            ResnetBlock(1, 64, 64, 2, bn),
            ResnetBlock(2, 128, 64, 2, bn),
            ResnetBlock(2, 256, 128, 2, bn),
            ResnetBlock(2, 512, 256, 2, bn))

        self.fcout = nn.Linear(512, 10)

    def forward(self, x):
        out = self.layer1(x)
        out = self.blocks(out)
        out = out.reshape(out.size(0), -1)
        out = self.fcout(out)
        return out

num_epochs = 50
num_classes = 10
batch_size = 50
learning_rate = 0.00001

trans = transforms.ToTensor()
train_dataset = torchvision.datasets.CIFAR10(root="./dataset_pytorch", train=True, download=True, transform=trans)
test_dataset = torchvision.datasets.CIFAR10(root="./dataset_pytorch", train=False, download=True, transform=trans)

train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)

def weights_init(m):
    if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
        nn.init.xavier_uniform_(m.weight.data)
        nn.init.zeros_(m.bias.data)

model = ConvNet()
model.apply(weights_init)

model.to(device)

summary(model, (3,32,32))

# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, eps=1e-6)

# Train the model
total_step = len(train_loader)
loss_list = []
acc_list = []
acc_list_test = []
for epoch in range(num_epochs):
    total = 0
    correct = 0
    for i, (images, labels) in enumerate(train_loader):
        images = images.to(device)
        labels = labels.to(device)

        optimizer.zero_grad()

        # Run the forward pass
        outputs = model(images)
        loss = criterion(outputs, labels)
        loss_list.append(loss.item())

        # Backprop and perform Adam optimisation
        loss.backward()
        optimizer.step()

        # Track the accuracy
        total += labels.size(0)
        _, predicted = torch.max(outputs.data, 1)
        correct += (predicted == labels).sum().item()
    acc_list.append(correct / total)

    print("Train")
    print('Epoch [{}/{}], Accuracy: {:.2f}%'
          .format(epoch + 1, num_epochs, (correct / total) * 100))

    total_test = 0
    correct_test = 0
    for i, (images, labels) in enumerate(test_loader):
        images = images.to(device)
        labels = labels.to(device)

        # Run the forward pass
        outputs = model(images)

        # Track the accuracy
        total_test += labels.size(0)
        _, predicted = torch.max(outputs.data, 1)
        correct_test += (predicted == labels).sum().item()
    acc_list_test.append(correct_test / total_test)
    print("Test")
    print('Epoch [{}/{}], Accuracy: {:.2f}%'
          .format(epoch + 1, num_epochs, (correct_test / total_test) * 100))

It's weird because it's throwing me that error Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same even though I've moved both the model and the data to cuda.
I guess it's related with how I defined or used "ResnetBlock", because if I remove from ConvNet those blocks (removing the line out = self.blocks(out)), the code works. But I don't know what I'm doing wrong.

Output:
The problem is in this line:
model.to(device)

to is not in-place. It returns the converted model. You need to change it to:
model = model.to(device)

EDIT: Another problem: vanilla list cannot be tracked by PyTorch. You need to use nn.ModuleList.
From
self.layers = []
self.relus = []

To
self.layers = nn.ModuleList()
self.relus = nn.ModuleList()


Instruction: How to get the total number of batch iteration from pytorch dataloader?

Input:
I have a question that How to get the total number of batch iteration from pytorch dataloader?
The following is a common code for training
for i, batch in enumerate(dataloader):

Then, is there any method to get the total number of iteration for the "for loop"?
In my NLP problem, the total number of iteration is different from int(n_train_samples/batch_size)...
For example, if I truncate train data only 10,000 samples and set the batch size as 1024, then 363 iteration occurs in my NLP problem.
I wonder how to get the number of total iteration in "the for-loop".
Thank you.

Output: len(dataloader) returns the total number of batches. It depends on the __len__ function of your dataset, so make sure it is set correctly.


Instruction: Gradient accumulation in an RNN

Input:
I ran into some memory issues (GPU) when running a large RNN network, but I want to keep my batch size reasonable so I wanted to try out gradient accumulation. In a network where you predict the output in one go, that seems self-evident but in an RNN you do multiple forward passes for each input step. Because of that, I fear that my implementation does not work as intended. I started from user albanD's excellent examples here , but I think they should be modified when using an RNN. The reason I think that is because you accumulate much more gradients because you do multiple forwards per sequence.
My current implementation looks like this, at the same time allowing for AMP in PyTorch 1.6 which seems important - everything needs to be called in the right place. Note that this is just an abstract version, which might seem like a lot of code but it is mostly comments.
def train(epochs):
    """Main training loop. Loops for `epoch` number of epochs. Calls `process`."""
    for epoch in range(1, epochs + 1):
        train_loss = process("train")
        valid_loss = process("valid")
        # ... check whether we improved over earlier epochs
        if lr_scheduler:
            lr_scheduler.step(valid_loss)

def process(do):
    """Do a single epoch run through the dataloader of the training or validation set.
       Also takes care of optimizing the model after every `gradient_accumulation_steps` steps.
       Calls `step` for each batch where it gets the loss from."""
    if do == "train":
        model.train()
        torch.set_grad_enabled(True)
    else:
        model.eval()
        torch.set_grad_enabled(False)

    loss = 0.
    for batch_idx, batch in enumerate(dataloaders[do]):
        step_loss, avg_step_loss = step(batch)
        loss += avg_step_loss

        if do == "train":
            if amp:
                scaler.scale(step_loss).backward()

                if (batch_idx + 1) % gradient_accumulation_steps == 0:
                    # Unscales the gradients of optimizer's assigned params in-place
                    scaler.unscale_(optimizer)
                    # clip in-place
                    clip_grad_norm_(model.parameters(), 2.0)
                    scaler.step(optimizer)
                    scaler.update()
                    model.zero_grad()
            else:
                step_loss.backward()
                if (batch_idx + 1) % gradient_accumulation_steps == 0:
                    clip_grad_norm_(model.parameters(), 2.0)
                    optimizer.step()
                    model.zero_grad()

        # return average loss
        return loss / len(dataloaders[do])

    def step():
        """Processes one step (one batch) by forwarding multiple times to get a final prediction for a given sequence."""
        # do stuff... init hidden state and first input etc.
        loss = torch.tensor([0.]).to(device)

        for i in range(target_len):
            with torch.cuda.amp.autocast(enabled=amp):
                # overwrite previous decoder_hidden
                output, decoder_hidden = model(decoder_input, decoder_hidden)

                # compute loss between predicted classes (bs x classes) and correct classes for _this word_
                item_loss = criterion(output, target_tensor[i])

                # We calculate the gradients for the average step so that when
                # we do take an optimizer.step, it takes into account the mean step_loss
                # across batches. So basically (A+B+C)/3 = A/3 + B/3 + C/3
                loss += (item_loss / gradient_accumulation_steps)

            topv, topi = output.topk(1)
            decoder_input = topi.detach()

        return loss, loss.item() / target_len

The above does not seem to work as I had hoped, i.e. it still runs into out-of-memory issues very quickly. I think the reason is that step already accumulates so much information, but I am not sure.

Output:
For simplicity, I will only take care of amp enabled gradient accumulation, without amp the idea is the same. And your step presented runs under amp so let's stick to that.
step
In PyTorch documentation about amp you have an example of gradient accumulation. You should do it inside step. Each time you run loss.backward() gradient is accumulated inside tensor leafs which can be optimized by optimizer. Hence, your step should look like this (see comments):
def step():
    """Processes one step (one batch) by forwarding multiple times to get a final prediction for a given sequence."""
    # You should not accumulate loss on `GPU`, RAM and CPU is better for that
    # Use GPU only for calculations, not for gathering metrics etc.
    loss = 0

    for i in range(target_len):
        with torch.cuda.amp.autocast(enabled=amp):
            # where decoder_input is from?
            # I assume there is one in real code
            output, decoder_hidden = model(decoder_input, decoder_hidden)
            # Here you divide by accumulation steps
            item_loss = criterion(output, target_tensor[i]) / (
                gradient_accumulation_steps * target_len
            )

        scaler.scale(item_loss).backward()
        loss += item_loss.detach().item()

        # Not sure what was topv for here
        _, topi = output.topk(1)
        decoder_input = topi.detach()

    # No need to return loss now as we did backward above
    return loss / target_len

As you detach decoder_input anyway (so it is like totally new hidden input without history and parameters will be optimized based on that, not based on all runs) there is no need for backward in process. Also, you probably don't need decoder_hidden, if it isn't passed to the network, torch.tensor filled with zeros is passed implicitly.
Also we should divide by gradient_accumulation_steps * target_len as that's how many backwards we will run before single optimization step.
As some of your variables are ill-defined I assume you just made a scheme of what's going on.
Also, if you want the history to be kept you shouldn't detach decoder_input, in this case it would look like this:
def step():
    """Processes one step (one batch) by forwarding multiple times to get a final prediction for a given sequence."""
    loss = 0

    for i in range(target_len):
        with torch.cuda.amp.autocast(enabled=amp):
            output, decoder_hidden = model(decoder_input, decoder_hidden)
            item_loss = criterion(output, target_tensor[i]) / (
                gradient_accumulation_steps * target_len
            )

        _, topi = output.topk(1)
        decoder_input = topi

        loss += item_loss
    scaler.scale(loss).backward()
    return loss.detach().cpu() / target_len

This effectively goes through RNN multiple times and will probably raise OOM, not sure what you are after here. If that's the case there's not much you can do AFAIK as the RNN computations are simply too long to fit into the GPU.
process
Only relevant part of this code is presented, so it would be:
loss = 0.0
for batch_idx, batch in enumerate(dataloaders[do]):
    # Here everything is detached from graph so we're safe
    avg_step_loss = step(batch)
    loss += avg_step_loss

    if do == "train":
        if (batch_idx + 1) % gradient_accumulation_steps == 0:
            # You can use unscale as in the example in PyTorch's docs
            # just like you did
            scaler.unscale_(optimizer)
            # clip in-place
            clip_grad_norm_(model.parameters(), 2.0)
            scaler.step(optimizer)
            scaler.update()
            # IMO in this case optimizer.zero_grad is more readable
            # but it's a nitpicking
            optimizer.zero_grad()

# return average loss
return loss / len(dataloaders[do])

Question-like

[...] in an RNN you do multiple forward passes for each input step.
Because of that, I fear that my implementation does not work as
intended.

It does not matter. For each forward you should usually do one backward (seems to be the case here, see steps for possible options). After that we (usually) don't need loss connected to graph as we already performed backpropagation, got our gradients and are ready to optimize parameters.

That loss needs to have history, as it goes back to the process loop
where backward will be called on it.

No need to call backward in process as presented.


Instruction: IPython Widgets Separated by Character Rather Than Word

Input:
As you can tell in the picture below, the table is separated by characters rather than full words.
def apply(f):
    text = f
    text = re.sub(r'\W+', ' ', text)
    res = LM().check_probabilities(text, topk=20)
    l = str(res)
    paraphrase_widget = widgets.SelectMultiple(
    options=l,
    description='Paraphrases',
    disabled=False,
    layout= widgets.Layout(width='100%')
   )
    display(paraphrase_widget)
    return {"result": res}

apply("In order to")

Output:
The issue here is in unpacking pytorch's prediction and passing those results to the widget in proper format (a list of tuples). Here's how you can do that:
# Modify your widget to the following
paraphrase_widget = widgets.SelectMultiple(
      options=res['pred_topk'][2],
      description='Paraphrases',
      disabled=False,
      layout= widgets.Layout(width='100%', height="300px")
   )

Here's what this looks like for me:


Instruction: Generate random vectors with a given (numerical) distribution matrix

Input:
I'm trying to come up with a fast and smart way of generating random vectors from a distribution matrix, much like what is being discussed here:
Generate random numbers with a given (numerical) distribution
But the key difference is that I have a distribution matrix, rather than just a single vector.
Now obviously I could just create a for loop and loop over each vector in my matrix, but that doesn't seem very pythonic, or fast, so I'm kinda hoping that there is a better way of doing it.
To give a quick example of what I want to do:
Given a one-probability matrix
p = [[0.2, 0.4, 0.4],[0.1, 0.7, 0.2],[0.44, 0.5, 0.06],...]

I wish to draw elements, where each element gets selected with the probability in the probability matrix. (Essentially I want to generate a one-hot encoding from my one-probability matrix).
Which could for instance look like this given the above probabilities:
t = [2,1,2,...]

I need to do this for long sequences, and I need to do it millions of times, but only 1 time for each sequence each time. (Data augmentation for deep learning)
Does anyone have a good way of doing this?

Output:
You could use inverse transform sampling. Compute a cumulative distribution on your p matrix, sample a single random vector of size the height of the matrix, then return the largest index along each row of the cumulative matrix. In code:
p = np.array([[0.2, 0.4, 0.4],[0.1, 0.7, 0.2],[0.44, 0.5, 0.06]])
u = np.random.rand(p.shape[0])
idxs = (p.cumsum(1) < u).sum(1)

then the idxs will be sampled according to the rows of p. e.g.:
np.histogram((p[0].cumsum() < np.random.rand(10000,1)).sum(1), bins=3)
# array([1977, 4018, 4005]), ...


Instruction: "@" for tensor multiplication using pytorch

Input:
This article https://towardsdatascience.com/understand-kaiming-initialization-and-implementation-detail-in-pytorch-f7aa967e9138 about intelligent weights initialization uses the syntax
x@w

to signify tensor (/matrix) multiplication.  I had not seen that before and instead had presumed we would need to "spell it out" as :
 torch.mm(x, w.t())

What is required to use the former (nicer) syntax? That article did not show a complete set of the imports they were using.

Output:
Just Python 3.5 and above can use this "@" syntax. The following are equivalent:
a = torch.rand(2,2)
b = torch.rand(2,2)

c = a.mm(b)
print(c)

c = torch.mm(a, b)
print(c)

c = torch.matmul(a, b)
print(c)

c = a @ b # python > 3.5+
print(c)

Output:
tensor([[0.2675, 0.8140],
        [0.0415, 0.1644]])
tensor([[0.2675, 0.8140],
        [0.0415, 0.1644]])
tensor([[0.2675, 0.8140],
        [0.0415, 0.1644]])
tensor([[0.2675, 0.8140],
        [0.0415, 0.1644]])

I like to use mm syntax for matrix to matrix multiplication and mv for matrix to vector multiplication.
To get the transposed matrix I like to use easy a.T syntax.
One more thing to add:
a = torch.rand(2,2,2)
b = torch.rand(2,2,2)

c = torch.matmul(a, b)
print(c)

c = a @ b # python > 3.5+
print(c)

Output:
tensor([[[0.2951, 0.3021],
         [0.8663, 1.0430]],

        [[0.2674, 1.3792],
         [0.0895, 0.9703]]])
tensor([[[0.2951, 0.3021],
         [0.8663, 1.0430]],

        [[0.2674, 1.3792],
         [0.0895, 0.9703]]])

mm cannot work for rank>2 (tensor of rank 3 or more). So if you work with bigger ranks use just these: matmul or @ notation.


Instruction: How to mutate weights of a NN in pytorch

Input:
Im playing around with genetic algorithms with pytorch and I'm looking for a more efficient way of mutating the weights of the network (applying a small modification to them)
Right now I have a suboptimal solution where I loop through the parameters and apply a random modification.
child_agent = network()
for param in child_agent.parameters():
        if len(param.shape) == 4:  # weights of Conv2D
            for i0 in range(param.shape[0]):
                for i1 in range(param.shape[1]):
                    for i2 in range(param.shape[2]):
                        for i3 in range(param.shape[3]):
                            param[i0][i1][i2][i3] += mutation_power * np.random.randn()
        elif len(param.shape) == 2:  # weights of linear layer
            for i0 in range(param.shape[0]):
                for i1 in range(param.shape[1]):
                    param[i0][i1] += mutation_power * np.random.randn()
        elif len(param.shape) == 1:  # biases of linear layer or conv layer
            for i0 in range(param.shape[0]):
                param[i0] += mutation_power * np.random.randn()

This solution is bound to my architecture and needs recoding If I decide to add more layers. Is there any way to do this more efficiently and clean? Preferable that it works regardless of how the architecture of my network looks like.
Thanks

Output:
pytorch and numpy are tensor oriented, e.g. you make operations on multiple items contained within multidimensional array-like object.
You can change your whole code to this single line:
import torch

child_agent = network()
for param in child_agent.parameters():
    param.data += mutation.power * torch.randn_like(param)

randn_like (docs here) creates random normal tensor with the same shape as param.
Also, if this parameter requires grad (which it probably does), you should modify it's data field.
MCVE:
import torch

mutation_power = 0.4

child_agent = torch.nn.Sequential(
    torch.nn.Conv2d(1, 3, 3, padding=1), torch.nn.Linear(10, 20)
)

for param in child_agent.parameters():
    param.data += mutation_power * torch.randn_like(param)


Instruction: cnn wrong prediction even though model shows good accuracy in training and validation data

Input:
I have used the skin cancer classification competition data in Kaggle. There are 4 labels and the entire data is imbalanced. I ran the resnet 18 model on a 10 fold cross validation split to train the data and each fold was given around 2 epochs. The code has been attached below.
Basically the model gave 98.2% accuracy with 0.07 loss value in the train data and 98.1% accuracy and 0.06 loss value in the validation data. So this seemed pretty good.
However the problem is...prediction.py(code attached below). When I tried to predict, the model keeps giving the result as [0]. Even if it's a train image data.
Is there something wrong with my code?
Expected result:
if the image is the input, the output should be either 0,1,2 or 3
model.py(where the training happens)
import pandas as pd
import numpy as np

import torch
import torch.nn as nn

import os

import cv2

import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

import albumentations as A

from torch.utils.data import TensorDataset, DataLoader,Dataset
from torchvision import models
from collections import defaultdict
from torch.utils.data.sampler import RandomSampler
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn import model_selection
from tqdm import tqdm
import gc

# generate data from csv file
class Build_dataset(Dataset):
    def __init__(self, csv, split, mode, transform=None):
        self.csv = csv.reset_index(drop=True)
        self.split = split
        self.mode = mode
        self.transform = transform

    def __len__(self):
        return self.csv.shape[0]

    def __getitem__(self, index):
        row = self.csv.iloc[index]

        image = cv2.imread(row.filepath)
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image=image)
            image = res['image'].astype(np.float32)
        else:
            image = image.astype(np.float32)

        image = image.transpose(2, 0, 1)
        data = torch.tensor(image).float()

        if self.mode == 'test':
            return data
        else:
            return data, torch.tensor(self.csv.iloc[index].target).long()

# training data
def train_epoch(model, loader, optimizer,loss_fn,device, scheduler,n_examples):

    model = model.train()

losses = []
correct_predictions = 0

for inputs, labels in tqdm(loader):
    inputs = inputs.to(device)
    labels = labels.to(device)

    outputs = model(inputs)

    _, preds = torch.max(outputs, dim=1)
    loss = loss_fn(outputs, labels)

    correct_predictions += torch.sum(preds == labels)
    losses.append(loss.item())

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()
# here you delete inputs and labels and then use gc.collect
    del inputs, labels
    gc.collect()

return correct_predictions.double() / n_examples, np.mean(losses)

# validation data
def val_epoch(model, loader,loss_fn, device,n_examples):

    model = model.eval()

    losses = []
    correct_predictions = 0

    with torch.no_grad():
        for inputs, labels in tqdm(loader):
            inputs = inputs.to(device)
            labels = labels.to(device)
            outputs = model(inputs)
            _, preds = torch.max(outputs, dim=1)
            loss = loss_fn(outputs, labels)
            correct_predictions += torch.sum(preds == labels)
            losses.append(loss.item())
            # here you delete inputs and labels and then use gc.collect
            del inputs, labels
            gc.collect()

    return correct_predictions.double() / n_examples, np.mean(losses)

 def train(fold, model,device, num_epochs):

    df_train = df[df.kfold != fold].reset_index(drop=True)
    df_valid = df[df.kfold == fold].reset_index(drop=True)
    # generate data
    dataset_train = Build_dataset(df_train,  'train', 'train', transform=transforms_train)
    dataset_valid = Build_dataset(df_valid, 'train', 'val', transform=transforms_val)

    #load data
    train_loader = DataLoader(dataset_train, batch_size = 64,sampler=RandomSampler(dataset_train),
num_workers=4)
    valid_loader = DataLoader(dataset_valid, batch_size = 32,shuffle = True, num_workers= 4 )

    dataset_train_size = len(dataset_train)

    dataset_valid_size = len(dataset_valid)

    optimizer = optim.Adam(model.parameters(), lr = 1e-4)

    model = model.to(device)

    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, patience = 3,threshold = 0.001, mode =
'max')

    loss_fn = nn.CrossEntropyLoss().to(device)

    history = defaultdict(list)

    best_accuracy = 0.0

    for epoch in range(num_epochs):
        print(f'Epoch {epoch+1} / {num_epochs}')
        print ('-'*30)

        train_acc, train_loss = train_epoch(model, train_loader, optimizer, loss_fn, device,
scheduler, dataset_train_size)
        print(f'Train loss {train_loss} accuracy {train_acc}')
        valid_acc, valid_loss = val_epoch(model, valid_loader, loss_fn, device,dataset_valid_size)
        print(f'Val   loss {valid_loss} accuracy {valid_acc}')
        print()

        history['train_acc'].append(train_acc)
        history['train_loss'].append(train_loss)
        history['val_acc'].append(valid_acc)
        history['val_loss'].append(valid_loss)

        if valid_acc > best_accuracy:
            print('saving model')
            torch.save(model.state_dict(), f'best_model_{fold}.bin')
            best_accuracy = valid_acc

    print(f'Best Accuracy: {best_accuracy}')

    model.load_state_dict(torch.load(f'best_model_{fold}.bin'))

    return model, history

 if __name__ == '__main__':
    #competition data -2020
    data_dir = "../input/jpeg-melanoma-384x384"
    #competition data - 2019
    data_dir2 = "../input/jpeg-isic2019-384x384"
    # device
    device = torch.device("cuda")

    # augmenting images

    image_size = 384
    transforms_train = A.Compose([
        A.Transpose(p=0.5),
        A.VerticalFlip(p=0.5),
        A.HorizontalFlip(p=0.5),
        A.RandomBrightness(limit=0.2, p=0.75),
        A.RandomContrast(limit=0.2, p=0.75),
        A.OneOf([
            A.MedianBlur(blur_limit=5),
            A.GaussianBlur(blur_limit=5),
            A.GaussNoise(var_limit=(5.0, 30.0)),
        ], p=0.7),

        A.OneOf([
            A.OpticalDistortion(distort_limit=1.0),
            A.GridDistortion(num_steps=5, distort_limit=1.),
            A.ElasticTransform(alpha=3),
        ], p=0.7),

        A.CLAHE(clip_limit=4.0, p=0.7),
        A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),
        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),
        A.Resize(image_size, image_size),
        A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1,
p=0.7),
        A.Normalize()
    ])

    transforms_val = A.Compose([
        A.Resize(image_size, image_size),
        A.Normalize()
    ])
    # create data
    df_train = pd.read_csv(os.path.join(data_dir, "train.csv"))  #/kaggle/input/siim-isic-melanoma-classification/train.csv
    df_train.head()

    df_train['is_ext'] = 0
    df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir, 'train', f'{x}.jpg'))

    # dataset from 2020 data
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule', 'unknown'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation', 'unknown'))

    # shuffle data
    df = df_train.sample(frac=1).reset_index(drop=True)

    # creating 8 different target values
    new_target = {d: idx for idx, d in enumerate(sorted(df.diagnosis.unique()))}
    df['target'] = df['diagnosis'].map(new_target)
    mel_idx = new_target['melanoma']

    # creating 10 fold cross validation data
    df = df_train.sample(frac=1).reset_index(drop=True)
    df['kfold'] = -1
    y = df_train.target.values
    kf = model_selection.StratifiedKFold(n_splits=10,shuffle=True)
    idx = kf.get_n_splits(X=df,y=y)
    print(idx)
    for fold,(x,y) in enumerate(kf.split(X=df,y=y)):
        df.loc[y,'kfold'] = fold

    df = df[['filepath','diagnosis', 'target', 'is_ext', 'kfold']]

    class_names = list(df['diagnosis'].unique())

    # create model

    def create_model(n_classes):
        model = models.resnet18(pretrained=True)

        n_features = model.fc.in_features
        model.fc = nn.Linear(n_features, n_classes)
        return model.to(device)

    base_model = create_model(len(class_names)) # model ready

    # run the model
    for i in range(10):
        #train
        base_model, history = train(i, base_model, device, num_epochs = 2) # train data

prediction.py
from torchvision import models
import torch
import torch.nn as nn
import albumentations as A
import cv2
import os
import numpy as np

device = torch.device("cuda")
MODEL = None
MODEL_PATH = "../input/prediction/best_model_4.bin"

def create_model(n_classes):
    model = models.resnet18(pretrained=True)

    n_features = model.fc.in_features
    model.fc = nn.Linear(n_features, n_classes)
    return model.to(device)
# generate the data to tensor with transform application

# converting the image to tensor by using the transforms function

class get_image:
    def __init__(self, image_path, targets, transform = None):
        self.image_path = image_path
        self.targets = targets
        self.transform = transform

    def __len__(self):
        return len(self.image_path)
    def __getitem__(self, item):
        targets = self.targets[item]
        resize = 384
        image = cv2.imread(self.image_path[item])
        image = cv2.resize(image, (resize, resize))
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image = image)
            image = res['image'].astype(np.float32)
        image = image.transpose(2, 0, 1)
        data = torch.tensor(image).float()
        targets = torch.tensor(targets)

        return data, targets

# load the data by using torch data
# predict values

# predict function
def predict(image_path, model, model_path):
    image_size = 384

    transforms_val = A.Compose([
        A.Resize(image_size, image_size),
        A.Normalize()
    ])

    test_images = [image_path]
    test_targets = [0]

    test_data = get_image(
        image_path = test_images,
        targets = test_targets,
        transform=transforms_val)
    # loading the data
    test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=1, shuffle = False,
num_workers=0)
    model = create_model(n_classes = 4)
    model.load_state_dict(torch.load(model_path))
    model.to(device)
    model.eval()
    prediction = []

    with torch.no_grad():
        for test_data, test_target in test_dataloader:
            test_data = test_data.to(device)
            test_target = test_target.to(device)

            outputs = model(test_data)
            _,preds = torch.max(outputs.cpu(), 1)

            #prediction.extend(preds)

            prediction = np.vstack((preds)).ravel()

            return prediction

def upload_predict():
    image_file = "../input/whatever/ISIC_0075663.jpg"

    if image_file:
        pred = predict(image_file, MODEL, MODEL_PATH)
        print(pred)

    return pred

the label and it's count is given right here
3    27126
2     5193
1      584
0      223

Here 0 is considered malignant type cancer and the other labels are of different types.
Here is the link to the data: https://www.kaggle.com/cdeotte/jpeg-melanoma-384x384

Output:
I think you might have the answer to your question! You said:

There are 4 labels and the entire data is imbalanced

Assuming that label 0 is no cancer and 1, 2, 3 are cases with different types of skin cancer. If you said that prediction classes are imbalanced, I'm guessing that 98% of the entire sample is 0, so your algorithm simply predicts every case to be 0 so that it will get right 98% of the time. When your algorithm gets to your test set, it will simply predict everything to be 0.
So the problem isn't with your code. You must balance your dataset by upsampling minority classes, downsampling majority class, assigning a weight/bias to your data or using some sort of model ensemble see https://elitedatascience.com/imbalanced-classes. Check out the credit card fraud detection tutorials such as https://towardsdatascience.com/credit-card-fraud-detection-1b3b3b44109b.


Instruction: How to transform pytorch tensor to view such as ((value0, row0, column0), (value1, row0, column1)...)?

Input:
How do I transform PyTorch tensor to view such as ((value0, row0, column0), (value1, row0, column1)...)? For example, if I have a tensor
a = torch.tensor([[5, 7],
                  [9, 11]])

The function should return:
((5, 0, 0), (7, 0, 1), (9, 1, 0), (11, 1, 1)). Also, I want it function to be applied for constants, 1D, 3D, 4D, etc. tensors, e.g. for 3D I want it to output (value0, dim1, dim2, dim3).
I tried to use for loops but didn't realize how to use them for this task: I can make a function only for a constant number of dimensions, such as only for 2D.

Output:
welcome to StackOverflow !
If the assumption that your tensor contains only non-zero values, then I guess the easiest way is to use to_sparse :
a = torch.tensor([[5, 7], [9, 11]])
def coords(t):
    sp_t = t.to_sparse()
    idx = sp_t.indices().T
    vals = sp_t.values()
    return torch.cat([vals.view(-1,1), idx],1)
coords(a)
>>> tensor([[ 5,  0,  0],
    [ 7,  0,  1],
    [ 9,  1,  0],
    [11,  1,  1]])

If that cannot be assumed, please tell me, a workaround with repeat and repeat_interleave should be possible


Instruction: Using AWS Sagemaker for model performance without creating endpoint

Input:
I've been using Amazon Sagemaker Notebooks to build a pytorch model for an NLP task.
I know you can use Sagemaker to train, deploy, hyper parameter tuning, and model monitoring.
However, it looks like you have to create an inference endpoint in order to monitor the model's inference performance.
I already have a EC2 instance setup to perform inference tasks on our model, which is currently on a development box and rather not use an endpoint to make
Is it possible to use Sagemaker to train, run hyperparam tuning and model eval without creating an endpoint.

Output:
If you don't want to keep an inference endpoint up, one option is to use SageMaker Processing to run a job that takes your trained model and test dataset as input, performs inference and computes evaluation metrics, and saves them to S3 in a JSON file.
This Jupyter notebook example steps through (1) preprocessing training and test data, (2) training a model, then (3) evaluating the model