Instruction: Pytorch training script on slurm doesn't write on stdout

Input:
I recently moved from tensorflow to pytorch, and today I had a little problem.
I run my scripts with slurm on an hpc, and all the stdout is redirected to a text file. The problem is that from this afternoon it stopped writing to that file while training (it should write training progresses), but the training is still going on, since slurm job is executing, and it saves model checkpoints every epoch.
The code of training loop is the following:
class TrainingParams:
    def __init__(self, model_params):
        self.BATCH_SIZE = 320
        self.LEARNING_RATE = 6e-5
        self.EPOCHS = 8
        with open('weights/full2021/classes/classes_weights.json', 'r') as fp:
            classes_weights = json.load(fp)
            classes_weights = torch.as_tensor(list(classes_weights.values()), dtype=torch.float)
        self.loss = nn.CrossEntropyLoss(weight=classes_weights)
        self.optimizer = torch.optim.Adamax(model_params, lr=self.LEARNING_RATE)

def get_pbar(progress):
    trattini = []
    progress = math.ceil(100)
    for i in range(0, math.floor(progress / 4)):
        trattini.append('-')
    for i in range(0, 25 - (math.floor(progress / 4))):
        trattini.append(' ')
    pbar = '[' + ''.join(trattini) + ']'
    return pbar

def train(model: nn.Module, train_paths: dict, val_paths: dict,
          resume=False, units=None, starting_epoch=1):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)
    model.cuda()
    if resume:
        checkpoint = torch.load(f"weights/full2021/unfreezed_cp_{units}/checkpoint_epoch-{starting_epoch - 1}")
        model.load_state_dict(checkpoint['model'])
        training_params = TrainingParams(model.parameters())
        optimizer = training_params.optimizer
        optimizer.load_state_dict(checkpoint['optimizer'])
        scaler = torch.cuda.amp.GradScaler()
        scaler.load_state_dict(checkpoint['scaler'])
        print(f'Correctly restored checkpoint of network with units {units} at epoch {starting_epoch - 1}')
    else:
        training_params = TrainingParams(model.parameters())
        optimizer = training_params.optimizer
        scaler = torch.cuda.amp.GradScaler()
    loss = training_params.loss.cuda()

    train_data = Dataset(train_paths['ids'], train_paths['attention'], train_paths['labels'])
    val_data = Dataset(val_paths['ids'], val_paths['attention'], val_paths['labels'])
    train_len = len(train_data)
    val_len = len(val_data)
    train_data = torch.utils.data.DataLoader(train_data, batch_size=training_params.BATCH_SIZE, shuffle=True)
    val_data = torch.utils.data.DataLoader(val_data, batch_size=training_params.BATCH_SIZE)

    history = {
        'epoch': [],
        'loss': [],
        'accuracy': [],
        'val_loss': [],
        'val_accuracy': []
    }

    print(f'Starting training from epoch {starting_epoch}...')

    for epoch in range(0, training_params.EPOCHS):
        start_time = time.time()
        train_accuracy = 0
        train_loss = 0
        val_accuracy = 0
        val_loss = 0
        for i, data in enumerate(train_data, 0):
            input, label = data

            model.zero_grad(set_to_none=True)

            with torch.cuda.amp.autocast():
                label = label.long().to(device)
                input_ids = input['input_ids'].squeeze(1).to(device)
                attention_mask = input['attention_mask'].to(device)

                output = model(input_ids, attention_mask)

                batch_loss = loss(output, label)

            scaler.scale(batch_loss).backward()

            scaler.unscale_(optimizer)

            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=3.0)

            scaler.step(optimizer)

            scaler.update()

            train_loss += batch_loss.item()

            batch_accuracy = (output.argmax(dim=1) == label).sum().item()
            train_accuracy += batch_accuracy

            progress = 100 * i / (train_len // training_params.BATCH_SIZE)
            print(f'Epoch: {starting_epoch + epoch} | '
                  f'It: {i: 5d}/{train_len // training_params.BATCH_SIZE} | '
                  f'Elapsed: {time.time() - start_time: .2f}s | '
                  f'Progress: {get_pbar(progress)} {progress: .2f}% | '
                  f'Loss: {train_loss / (i + 1): .3f} | '
                  f'Accuracy: {train_accuracy / (training_params.BATCH_SIZE * (i + 1)): .3f} |'
            )

        with torch.no_grad():
            for input, label in val_data:
                with torch.cuda.amp.autocast():
                    label = label.long().cuda()
                    input_ids = input['input_ids'].squeeze(1).cuda()
                    attention_mask = input['attention_mask'].cuda()

                    output = model(input_ids, attention_mask)

                    batch_loss = loss(output, label)
                    val_loss += batch_loss.item()

                    batch_accuracy = (output.argmax(dim=1) == label).sum().item()
                    val_accuracy += batch_accuracy

        print_train_loss = train_loss / (train_len // training_params.BATCH_SIZE)
        print_train_accuracy = train_accuracy / train_len
        print_val_loss = val_loss / (val_len // training_params.BATCH_SIZE)
        print_val_accuracy = val_accuracy / val_len
        print(f'-----> Epoch {starting_epoch + epoch} ended. '
              f'Total elapsed time: {(time.time() - start_time) * 60: .2f}m | '
              f'| Training Loss: {print_train_loss: .3f} '
              f'| Training Accuracy: {print_train_accuracy: .3f} '
              f'| Validation Loss: {print_val_loss: .3f} '
              f'| Validation Accuracy: {print_val_accuracy: .3f}'
        )

        history['epoch'].append(starting_epoch + epoch)
        history['loss'].append(print_train_loss)
        history['accuracy'].append(print_train_accuracy)
        history['val_loss'].append(print_val_loss)
        history['val_accuracy'].append(print_val_accuracy)

        checkpoint = {"model": model.state_dict(),
                      "optimizer": optimizer.state_dict(),
                      "scaler": scaler.state_dict()}
        torch.save(checkpoint, f"weights/full2021/unfreezed_cp_{units}/checkpoint_epoch-{starting_epoch + epoch}")

    history = pd.DataFrame.from_dict(history)
    history.to_csv(f'history_log-dir/full2021/history_{units}_epoch-{starting_epoch + training_params.EPOCHS - 1}',
                   index=False)

Basically it skips all "print" statements, since I can see everytime these lines:
2022-06-15 12:08:28.989315: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
Some weights of the model checkpoint at Bert/ were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

I can't understand why this is happening, it always worked correctly since this afternoon.
PS: now it updated one of the output files (I'm training on two different nodes with two models), and it debugged ~2000 iterations in one shot, so it now debug only after tot iterations. This is strange because since this afternoon it was updating the text file at every iteration (e.g. if I was pressing f5 continuously, I was seeing the file in constant update).
What can be the cause?

Output: The print statements are being loaded into be buffer which then prints once its filled up. To force the print to be printed directly try running your code with the -u flag. Something like python -u my_code.py. Alternatively, adding flush=True to the print statements should accomplish the same.


Instruction: How to avoid "RuntimeError: CUDA out of memory." during inference of one single image?

Input:
I am facing the famous "CUDA out of memory" error.
File "DATA\instance-mask-r-cnn-torch\venv\lib\site-packages\torchvision\models\detection\roi_heads.py", line 416, in paste_mask_in_image
    im_mask = torch.zeros((im_h, im_w), dtype=mask.dtype, device=mask.device)
RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 2.00 GiB total capacity; 1.66 GiB already allocated; 0 bytes free; 1.73 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF

Windows 10, CUDA 11.3, torch 0.11.0+cu113, torchvision 0.12.0+cu113
On the ENV I played with PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:32, 128, 8, 24, 32... without success.
An image of size 640x512 (1.5mb) works, another one of size 3264x1840 (1.75mb) leads to an OOME.
import torchvision.transforms
from torchvision.models.detection import mask_rcnn
import torch
from PIL import Image
import gc

if torch.cuda.is_available():
    print(f'GPU: {torch.cuda.get_device_name(0)}')
    device = torch.device('cuda')
    torch.cuda.empty_cache()
else:
    device = torch.device('cpu')

print(f'Device: {device}')

model = mask_rcnn.maskrcnn_resnet50_fpn(pretrained=True)
print(model.eval())

model.to(device)

img_path = 'images/tv_image05.png'
img_path = 'images/DJI_20220519110029_0001_W.JPG'
img_path = 'images/DJI_20220519110143_0021_T.JPG'
img_path = 'images/WP_20160104_09_52_53_Pro.jpg'

img = Image.open(img_path).convert("RGB")

img_tensor = torchvision.transforms.functional.to_tensor(img)

with torch.no_grad():
    predictions = model([img_tensor.cuda()])

print(predictions)

gc.collect()
torch.cuda.empty_cache()

So far i found lots of hints, reducing batch size. But I am not using training mode. what else can I do to be able to process images of sizes up to 7mb?

Output:
The 3264x1840 image is going to be 72MB in float32. Since it works for your 640x512 image, I'd suggest resizing it.
Simply add torchvision.transforms.functional.resize(img,512)
Another common trick is to quantize the model and the image to float16 but this may degrade the model accuracy depending on what you're doing.


Instruction: Will using image augmentation techniques in pytorch increase the dataset size on local machine also

Input:
I was training a custom model in pytorch and the dataset was very uneven. As in there are 10 classes for which some class have only 800 images while some have 4000 images. I found that image augmentation was a solution for my problem to avoid overfitting. But i got confused in between while implementing, the below codes were used to alter the features of the images
loader_transform = transforms.Compose([
transforms.RandomRotation(30),
transforms.RandomResizedCrop(140),
transforms.RandomHorizontalFlip()
])

but while training it shows the same original number of images where did the newly created augmented dataset go. And if i want to save it on my local machine and to make all classes even what can be done??

Output:
It looks like you are using online augmentations, If you like to use offline please do a pre-processing step that saves the images and then use them in the training step
Please make sure you understand the difference between online augmentations and offline augmentations
Offline or pre-processing Augmentation
To increase the size of the data set, enhancement is applied as a pre-processing step. Usually, we do this when we want to expand a small training data set.
When applying to larger data sets, we have to consider disk space
Online or real-time Augmentation
The augmentation is being applied in real-time through random augmentations.
Since the augmented images do not need to be saved on the disk, this method is usually applied to large data sets.
At each epoch, the online augmentation model will see a different image.


Instruction: PyTorch model always returns zeros while Keras model works fine

Input:
I am implementing the same (simple) model in tf.keras and PyTorch. My Keras model does fine, but my Torch model always predicts zeros and does poorly as a result.
My Keras model is defined as:
model = Sequential([
    Dense(5, activation='relu', name='layer1'),
    Dense(5, activation='relu', name='layer2'),
    Dense(1, activation='sigmoid', name='output')
])

model.compile(
    optimizer='adam',
    loss='binary_crossentropy',
    metrics=['accuracy']
)

history = model.fit(
    data.x_train,
    data.y_train,
    batch_size=128,
    epochs=30,
    validation_data=(data.x_test, data.y_test)
)

Meanwhile, my Torch model is defined like so:
class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.model = nn.Sequential(
            nn.Linear(19, 5),
            nn.ReLU(),
            nn.Linear(5, 5),
            nn.ReLU(),
            nn.Linear(5, 1),
            nn.Sigmoid()
        )

    def forward(self, x):
        return self.model(x)

def train(train_ds, test_ds, model):
    train_dl = DataLoader(train_ds, batch_size=128, shuffle=True)
    test_dl = DataLoader(test_ds, batch_size=128, shuffle=True)
    opt = optim.Adam(model.parameters())
    loss_func = F.binary_cross_entropy

    # 30 epochs
    for epoch in range(30):
        for xb, yb in train_dl:
            pred = model(xb)
            loss = loss_func(pred.reshape((pred.shape[0])), yb)

            loss.backward()
            opt.step()
            opt.zero_grad()

Both use default initialization, which is Glorot Uniform; the data passed to both is the same too. As far as I can tell, I've implemented the same model, with the same optimizer and hyper-parameters in both frameworks, but the PyTorch model returns all zeros when I print it out. What am I doing wrong?

Output:
Fixing the optimizer steps
For the optimizer to work in your Pytorch model, these are the steps you should take, it might be counter intuitive in the beginning, but you need to zero grad first:
opt.zero_grad()
loss.backward()
opt.step()

Evaluating different models
As a side note; If you want the models to be exactly the same you should validate that your basic Learning Rate and Optimizer parameters are the same. Same as the Weights initializations params, the method might be the same (Glorot) but it might be depended on some internal params, that you can fix in your initialization.


Instruction: Is the waveform the "raw" audio data? (Pytorch)

Input: I am using PyTorch in an audio deep learning project. I am using the torchaudio.load method to load the waveform and sample rate. Now, my question is, is the waveform considered the "raw" audio data? Is it the PCM data? If not, then how can I get PCM data from .ogg format?

Output:
 Solution
Yes, it's raw data.
For the explanation read below. If you know about sampling theory and how sound is generated, skip to the last paragraph.
 PCM
PCM is a fancy way to explain the process by which a continuous time wave is represented inside a computer. You can learn more in any introductory course/book of digital signal processing such chapter 3 of The Scientist and Engineer's Guide to Digital Signal Processing.
Briefly in a computer you can only represent finite quantities, so you need to take discrete samples in time (sampling) at a certain amplitudes (quantization).
When loading any audio file this process has been already done for you.
 RAW DATA
If you connect a speaker and you play a wave, the membrane will oscillate as the amplitude of such wave at every instant. This is the "raw" audio, a signal that contains the amplitude at each "time" instant. If you can "see" the wave changing with no discontinuity from left to right when plotting your data, it is very likely a raw vector.
What is non-raw data then? Every compression algorithm modifies the input vector with any sort of mathematical function, so that it occupies less space, but also is not understandable anymore by just looking at it. This is because the samples don't represent anymore an amplitude over time. If you'd play the compressed wave through a speaker you wouldn't get any sound, only noise.
 Pytorch
In the example you provided from the pytorch documentation we can clearly see that the plot represents raw data, sampled at 16kHz.
To exclude the possibility that

torchaudio.load could still give a sort of compressed object
the raw data is generated and plotted by plot_waveform

We can see that the waveform variable is long 54400 samples and sampled at 16kHz. This means it represents 54400*(1/16000) seconds, which are exactly 3.4s.
The plot shows 3.4seconds, thus telling us that what is represented in the variable waveform returned by the load function is the raw data.


Instruction: issue terminate called after throwing an instance of 'c10::CUDAError'

Input:
I am having issues with running yolov5 on colab. I was able to run the code fine when I had  I had more classes, and a slightly smaller dataset, but now I have decreased the amount of classes and 70 instances when the overall one has 3400 instances. Now I am getting this error.
terminate called after throwing an instance of 'c10::CUDAError'
Other times I will get
 cuda assertion index >= -sizes[i] && index < sizes[i] && index out of bounds
any idea what could be causing this and how to fix it?

Output: The issue was that I was not outputting any labels that I said existed. So for example if I claimed that there would be labels "0","1","2" in the training dataset. There was no instances of label "2".


Instruction: Convert function to exploit parallelization of the GPU

Input:
I have a function that uses values stored in one array to operate on another array.  This behaves similar to the numpy.hist function.  For example:
import numpy as np
from numba import jit

@jit(nopython=True)
def array_func(x, y, output_counts, output_weights):
    for row in range(x.size):
        col = int(x[row] * 10)
        output_counts[col] += 1
        output_weights[col] += y[row]

    return (output_counts, output_weights)

# in the current code these arrays exists ad pytorch tensors
# on the GPU and get converted to numpy arrays on the CPU before
# being passed to "array_func"
x = np.random.randint(0, 11, (1000)) / 10
y = np.random.randint(0, 100, (10000))

output_counts, output_weights = array_func(x, y, np.zeros(y.size), np.zeros(y.size))

While this works for arrays it does not work for torch tensors that are on the GPU.  This is close to what histogram functions do, but I also need the summation of binned values (i.e., the output_weights array/tensor).  The current function requires me to continually pass the data from GPU to CPU, followed by the CPU function being run in series.
Can this function be converted to run in parallel on the GPU?
##EDIT##
The challenge is caused by the following line:
output_weights[col] += y[row]

If it weren't for that line I could just use the torch.histc function.
Here's my thought: GPUs are "fast" because they have hundreds/thousands of threads available and can run parts of a big job (or many smaller jobs) on these threads.  However, if I convert the function above to work on torch tensors then there is no benefit to running on the GPU (it actually kills the performance).  I wonder if there is a way I can break of x so each value gets sent to different threads (similar to how apply_async does within multiprocessing)?
I'm open to other options.
In it's current form the function is fast, but the GPU-to-CPU data transfer is killing me.

Output:
Your computation is indeed a general histogram operation. There are multiple ways to compute this on a GPU regarding the number of items to scan, the size of the histogram and the distribution of the values.
For example, one solution consist in building local histograms in each separate kernel blocks and then perform a reduction. However, this solution is not well suited in your case since len(x) / len(y) is relatively small.
An alternative solution is to perform atomic updates of the histogram in parallel. This solutions only scale well if there is no atomic conflicts which is dependent of the actual input data. Indeed, if all value of x are equal, then all updates will be serialized which is slower than doing the accumulation sequentially on a CPU (due to the overhead of the atomic operations). Such a case is frequent on small histograms but assuming the distribution is close to uniform, this can be fine.
This operation can be done with Numba using CUDA (targetting Nvidia GPUs). Here is an example of kernel solving your problem:
@cuda.jit
def array_func(x, y, output_counts, output_weights):
    tx = cuda.threadIdx.x  # Thread id in a 1D block
    ty = cuda.blockIdx.x   # Block id in a 1D grid
    bw = cuda.blockDim.x   # Block width, i.e. number of threads per block
    pos = tx + ty * bw     # Compute flattened index inside the array
    if pos < x.size:
        col = int(x[pos] * 10)
        cuda.atomic.add(output_counts, col, 1)
        cuda.atomic.add(output_weights, col, y[pos])

For more information about how to run this kernel, please read the documentation. Note that the arrays output_counts and output_weights can possibly be directly created on the GPU so to avoid transfers. x and y should be on the GPU for better performance (otherwise a CPU reduction will be certainly faster). Also note that the kernel should be pretty fast so the overhead to run/wait it and allocate/free temporary array may be significant and even possibly slower than the kernel itself (but certainly faster than doing a double transfer from/to the CPU so to compute things on the CPU assuming data was on the GPU). Note also that such atomic accesses are only fast on quite recent Nvidia GPU that benefit from specific computing units for atomic operations.


Instruction: extracting tensor values given tensor index values torch

Input:
I have a tensor of values val with shape (b,n) and a tensor of indexes ind with shape (b,m) (where n>m). My goal is to take the values in val that corresponds to the indexes in ind. Ive tried using val[ind], but it only expanded the dimensions of val, rather than taking only the relevant items
val = torch.tensor([[1,2,3],
                    [4,5,6],
                    [7,8,9],
                    [10,11,12],
                    [13,14,15]])
ind = torch.tensor([[1,2],
                    [0,2],
                    [0,1],
                    [1,2],
                    [0,1]])
val[ind] # shaped (5,2,4), I need (5,2)

the wanted output is
torch.tensor([[2,3],
              [4,6],
              [7,8],
              [11,12],
              [13,14]])

Output:
You can perform such operation using torch.gather:
>>> val.gather(dim=1, index=ind)
tensor([[ 2,  3],
        [ 4,  6],
        [ 7,  8],
        [11, 12],
        [13, 14]])

Essentially indexing val's 2nd dimension using ind's values. The returned tensor out follows:
out[i][j] = val[i][ind[i]]


Instruction: How to sum the second column's values according to the first column's value in a pytorch tensor?

Input:
I've a tensor in pytorch, its first column's value choice is limited, but its second column's value is freely chosen, e.g:
val = torch.tensor([[1,233],
                    [1,222],
                    [2,333],
                    [2,3234],
                    [2,3242],
                    [2,3234],
                    [3,234],
                    [3,234],
                    [4,323]])

Now I want to sum all values in the second column if their correspoding first column's values are same, the output should be as following:
output_val=torch.tensor([[1,455],
                 [2,10043],
                 [3,468],
                 [4,323]])

I want to use pytorch's tensor-support APIs to hanlde this task instead of using python's for/while loop programming because I've more than billions of records to do this kind of handling, the for/while looping code will consume more than several days, and any suggestion is welcom.
Thanks!

Output:
Thanks for suggestions from @Shai and @Alexander-guyer, finally I've got the full solution to fully utilize the pytorch's parallel computing power（with its APIs） to do this kind of handling. The following is my final solution:
Input value tensor is:
val = torch.tensor([[1,233],
                    [1,222],
                    [2,333],
                    [2,3234],
                    [2,3242],
                    [2,3234],
                    [3,234],
                    [3,234],
                    [4,323]])

Now get its first and second columns into val0 and val1:
val0=val[:,0]
val1=val[:,1]

Now, use the torch.unique() to get its first column's unique values into uniq_val0, and get the inversal_index into index0:
uniq_val0, index0=torch.unique(val0, return_inverse=True)
zero_sum=torch.zeros(uniq_val0.shape, dtype=torch.int64)

Now, we could index_add_() to get the values' sum we want with the index0 we got from previous step:
output_val1=zero_sum.index_add_(0, index0, val1)

Now, we could stack the uniq_val0 and output_val1 togather, this is what we want:
output_val=torch.stack((uniq_val0, output_val1),-1)

Now, check the value, it's just what we want:
print(output_val)

tensor([[    1,   455],
        [    2, 10043],
        [    3,   468],
        [    4,   323]])


Instruction: Change of Tensor Dimension Cause an Error

Input:
I'm trying to test the principle of transforms.Resize while I find a confusing point. When I run the below codes:
import numpy as np
import torch
from torchvision import transforms

tim = np.array([[[1, 2, 3],
                 [1, 2, 3],
                 [1, 2, 3]],
                [[1, 2, 3],
                 [1, 2, 3],
                 [1, 2, 3]]]) # (2, 3, 3)

tim = torch.from_numpy(tim)

tf = transforms.Compose([  # Principle?
     transforms.ToPILImage(),
     transforms.Resize((6, 6)), # HW
     transforms.ToTensor()
])

mask = tf(tim)
squ = mask.squeeze()

A bug occurs:
Traceback (most recent call last):
  File "C:/Users/Tim/Desktop/U-Net/test.py", line 62, in <module>
    mask = tf(tim)
  File "C:\Users\Tim\.conda\envs\Segment\lib\site-packages\torchvision\transforms\transforms.py", line 95, in __call__
    img = t(img)
  File "C:\Users\Tim\.conda\envs\Segment\lib\site-packages\torchvision\transforms\transforms.py", line 227, in __call__
    return F.to_pil_image(pic, self.mode)
  File "C:\Users\Tim\.conda\envs\Segment\lib\site-packages\torchvision\transforms\functional.py", line 315, in to_pil_image
    raise TypeError(f"Input type {npimg.dtype} is not supported")
TypeError: Input type int32 is not supported

However, when I change the size of the tensor, the problem is solved:
tim = np.array([[[1, 2, 3],
                 [1, 2, 3],
                 [1, 2, 3]]]) # (1, 3, 3)

I’m wondering why this happens as the bug description has nothing to do with size but type. If anyone has any ideas on the reason, please let me know, thanks for your time!

Output:
Change the datatype to float ...
tim = np.array([[[1, 2, 3],
                 [1, 2, 3],
                 [1, 2, 3]],
                [[1, 2, 3],
                 [1, 2, 3],
                 [1, 2, 3]]], dtype=np.float32) # (2, 3, 3)

Make sure you know what the input data is.


Instruction: Pytorch - What is the proper way to minimize and maximize the same loss?

Input:
I have an objective function where I am looking to maximize a loss w.r.t f1 and f2, which are encoders; at the same time minimizing it w.r.t to g, which is a bijective convolution and X is just an image.

Here is how I assume it's supposed to be done.
obs_ch1, obs_ch23 = self.infomin.split_RGB_into_R_GB(obs=obs_RGB)
obs_ch1, obs_ch23 = self.infomin.R_GB_to_frame_stacked_R_GB(
    obs_R=obs_ch1, obs_GB=obs_ch23)

# minimize wrt g
logits = self.infomin.compute_logits(anchor=obs_ch1, pos=obs_ch23)
loss = self.cross_entropy_loss(logits, labels)

self.infomin_discrim_optimizer.zero_grad()
loss.backward()
self.infomin_discrim_optimizer.step()

# maximize wrt f1, f2
logits = self.infomin.compute_logits(
    anchor=obs_ch1.detach(), pos=obs_ch23.detach())
labels = torch.arange(logits.shape[0]).long().to(self.device)
loss = self.cross_entropy_loss(logits, labels)

self.infomin_encoders_optimizer.zero_grad()
(-loss).backward()
self.infomin_encoders_optimizer.step()

So I have both f1 and f2 using the same optimizer which is infomin_encoders_optimizer. I calculate the NCE loss w.r.t to just g then I detach that tensor and calculate NCE w.r.t f1 and f2. I take the opposite of that value and backpropagate them separately. The reason I do it this way is because I can't do them together because they are opposing objective directions. Also have to detach what comes out of g, because otherwise, it gives an error about multiple gradient updates to g.
How are you supposed to minimize and maximize an objective at the same time w.r.t to different parameters?

Output:
This is actually the proper way of proceeding in order to optimize a min-max objective. You can't solve this kind of problem with a single optimization step simply because both loss functions (for f1/f2 and for g) are based on a result computed with I_NCE. This means you are required to infer twice: first for computing the objective for g, then a second time for computing the objective for f1/f2.
Note this is a very similar procedure if not identical to training a generative adversarial network.


Instruction: TypeError with Dataloader

Input:
i used a very large dataset for testing my model. to make the testing samples fast, I would like to construct a data loader. but I'm getting errors. I couldn't solve it for two days. Here is my code:

 PRE_TRAINED_MODEL_NAME = 'bert-base-cased'
 tokenizer = BertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)

 class GPReviewDataset(Dataset):
    def __init__(self, Paragraph, target, tokenizer, max_len):
       self.Paragraph = Paragraph
       self.target= target
       self.tokenizer = tokenizer
       self.max_len = max_len

    def __len__(self):
       return len(self.Paragraph)

    def __getitem__(self, item):
       Paragraph = str(self.Paragraph[item])
       target = self.target[item]
       encoding = self.tokenizer.encode_plus(
       Paragraph,
       add_special_tokens=True,
       max_length=self.max_len,
       return_token_type_ids=False,
       pad_to_max_length=True,
       return_attention_mask=True,
       return_tensors='pt',
       )
       return {
       'review_text': Paragraph,
       'input_ids': encoding['input_ids'].flatten(),
       'attention_mask': encoding['attention_mask'].flatten(),
       'targets': torch.tensor(target, dtype=torch.long)
       }

def create_data_loader(df, tokenizer, max_len, batch_size):
    ds = GPReviewDataset(
    Paragraph=df.Paragraph.to_numpy(),
    target=df.target.to_numpy(),
    tokenizer=tokenizer,
    max_len=max_len
    )
   return DataLoader(
     ds,
    batch_size=batch_size,
    num_workers=4
    )

 # Main function
 paragraph=['Image to PDF Converter. ', 'Test Test']
 target=['0','1']
 df = pd.DataFrame({'Paragraph': paragraph, 'target': target})

 MAX_LEN='512'
 BATCH_SIZE = 1
 train_data_loader1 = create_data_loader(df, tokenizer, MAX_LEN, BATCH_SIZE)
 for d in train_data_loader1:
      print(d)

When I iterate over the dataloader  I got this error:
  TypeError: Caught TypeError in DataLoader worker process 0.
  Original Traceback (most recent call last):
  File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/worker.py",
  line 178, in _worker_loop
   data = fetcher.fetch(index)
  File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
  data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/usr/local/lib/python3.7/dist-packages/torch/utils/data/_utils/fetch.py", line 44, in <listcomp>
  data = [self.dataset[idx] for idx in possibly_batched_index]
  File "<ipython-input-3-c4f87a4dbb48>", line 20, in __getitem__
  return_tensors='pt',
  File "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils.py", line 1069, in encode_plus
    return_special_tokens_mask=return_special_tokens_mask,
  File "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils.py", line 1365, in prepare_for_model
    if max_length and total_len > max_length:
   TypeError: '>' not supported between instances of 'int' and 'str'

Can anyone help me? and Also, Can you give tips on how I can test my model on a large dataset? I mean what the faster way to test my model on 3M samples of data is?

Output:
The error is as it stated
File "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils.py", line 1365, in prepare_for_model
    if max_length and total_len > max_length:
   TypeError: '>' not supported between instances of 'int' and 'str'

You should change your MAX_LEN from string to int:
# MAX_LEN='512'
MAX_LEN=512


Instruction: train stable baselines 3 with examples?

Input:
For my basic evaulation of learning algorithms
I defined a custom environment.
Now with standard examples for stable baselines the learning
seems always to be initiated by stable baselines automatically
(by stablebaselines choosing random actions itsself and evaluating the rewards).
The standard learning seems to be done like this:
model.learn(total_timesteps=10000)
and this will try out different actions and optimize the action-observation-relations
while learning.
I would like to try out a really basic approach: for my custom environment I would
try to generate lists of examples, which actions should be taken according to some
situations of relevance (so there is a list of predefined observation-action-rewards).
And I would like to train the model with this list.
What would be the most appropriate way to implement this with stablebaselines3
(using pytorch)?
Additional information:
maybe the sense of the question could be compared to the idea in case of an atari game, not to always train a whole game sequence at once (from start to end of game, and then restart again until training ends), but instead to train the agent only with some more specific, representative situations of importance.
Or in chess: it seems to be a huge difference to let an agent
select randomly self choosen or randomly selected moves or
to let him follow moves played by masters in particular interesting
situations.
Maybe one could put the lists as main part of the environment reaction
(so e.g. train the agent with environment 1 for e.g. 1000 steps,
then train with environment 2 for e.g. 1000 steps and so on).
This could be a solution.
But the problem would be, that stable baselines would choose
actions itsself, so that it could not learn a complete sequence
of "correct" or like in chess masterful choosen steps in sequence.
So again the practical question is : is is possible, ot  important to bring stablebaselines to train  predefined actions instead of self choosen ones while training/learning?

Output:
Imitation Learning is essentially what you are looking for. There is an imitation library that sits on top of baselines that you can use to achieve this.
See this example on how to create a policy that mimics expert behavior to train the network. The behavior in this case comes from a set of action sequences or rollout. In the example the rollout comes from an expertly trained policy, but you can probably create a hand written one. See this on how to create a rollout.


Instruction: What these alphabets stands for?

Input:
Sorry if this question is trivial but I couldn't find the answer in the internet,
In pytorch documentation I saw these large alphabets in formulas
like:
Input:(N, C, H^in, W^out)
link: https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html
So what are these stands for and where
I can find the meaning of these alphabets.

Output:
Those are typical axis notations used in the PyTorch documentation to refer to the dimension sizes:

N: batch size
C: number of channels
W: the width of the tensor
H: the height of the tensor

Suffix _in and _out stand for the axis' corresponding input and output size respectively.


Instruction: Adding my custom loss messes up autograd in Pytorch?

Input:
I'm trying to use two different losses, MSELoss for some of my labels and a custom loss for the other labels. I'm then trying to sum these losses together before backprop. My model prints out the same loss after every epoch so I must be doing something wrong. Any help is appreciated! I suspect my implementation is messing up Pytorch's autograd. See code below:
mse_loss = torch.nn.MSELoss()
...
loss1 = mse_loss(preds[:,(0,1,3)], label[:,(0,1,3)])
print("loss1", loss1)
loss2 = my_custom_loss(preds[:,2], label[:,2])
print("loss2", loss2)
print("summing losses")
loss = sum([loss1, loss2]) # tensor + float = tensor
print("loss sum", loss)
loss = torch.autograd.Variable(loss, requires_grad=True)
print("loss after Variable(loss, requires_grad=True)", loss)

These print statements yield:
loss1 tensor(4946.1221, device='cuda:0', grad_fn=<MseLossBackward0>)
loss2 34.6672
summing losses
loss sum tensor(4980.7891, device='cuda:0', grad_fn=<AddBackward0>)
loss after Variable() tensor(4980.7891, device='cuda:0', requires_grad=True)

My custom loss function is below:
def my_custom_loss(preds, label):
    angle_diff = preds - label
    # /2 to bring angle diff between -180<theta<180
    half_angle_diff = angle_diff.detach().cpu().numpy()/2
    sine_diff = np.sin(half_angle_diff)
    square_sum = np.nansum(sine_diff**2)
    return square_sum

Output:
The reason why you are not backpropagating through your second loss is that you haven't defined it as a differentiable operator. You should stick with PyTorch operators without switching to NumPy.
Something like this will work:
def my_custom_loss(preds, label):
    half_angle_diff = (preds - label)/2
    sine_diff = torch.sin(half_angle_diff)
    square_sum = torch.nansum(sine_diff**2)
    return square_sum

You can check that your custom loss is differentiable with dummy inputs:
>>> preds = torch.rand(1,3,10,10, requires_grad=True)
>>> label = torch.rand(1,3,10,10)
>>> my_custom_loss(preds, label)
tensor(11.7584, grad_fn=<NansumBackward0>)

Notice the grad_fn attribute on it which shows the output tensor is indeed attached to a computational graph, and you can therefore perform back propagation from it.
Additionally, you should not use Variable as it is now deprecated.


Instruction: How to test my trained huggingface model on the test dataset?

Input:
I was following the huggingface tutorial on training a multiple choice QA model and trained my model with
training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=5e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=1,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_qa["train"],
    eval_dataset=tokenized_qa["validation"],
    tokenizer=tokenizer,
    data_collator=DataCollatorForMultipleChoice(tokenizer=tokenizer),
    compute_metrics=compute_metrics
)

trainer.train()

Afterwards I can load the model with:
# load trained model for testing
model = AutoModelForMultipleChoice.from_pretrained('results/checkpoint-1000')

But how can I test it on the testing dataset?
The dataset looks like:
DatasetDict({
    train: Dataset({
        features: ['id', 'sent1', 'sent2', 'ending0', 'ending1', 'ending2', 'ending3', 'label', 'input_ids', 'attention_mask'],
        num_rows: 10178
    })
    test: Dataset({
        features: ['id', 'sent1', 'sent2', 'ending0', 'ending1', 'ending2', 'ending3', 'label', 'input_ids', 'attention_mask'],
        num_rows: 1273
    })
    validation: Dataset({
        features: ['id', 'sent1', 'sent2', 'ending0', 'ending1', 'ending2', 'ending3', 'label', 'input_ids', 'attention_mask'],
        num_rows: 1272
    })
})

I have quite a bit of code so if there's more information needed please do let me know.

Output:
Okay figured it out and adding an answer for completion. Seems like the training arguments from the trainer class are not needed:
trainer = Trainer(
    model=model,
    tokenizer=tokenizer,
    data_collator=DataCollatorForMultipleChoice(tokenizer=tokenizer),
    compute_metrics=compute_metrics
)

Put in evaluation mode:
model.eval() # put in testing mode (dropout modules are deactivated)

And then call:
trainer.predict(tokenized_qa["test"])

PredictionOutput(predictions=array([[-1.284791 , -1.2848296, -1.2848794, -1.2848705],
       [-1.2848867, -1.2849237, -1.2848233, -1.2848446],
       [-1.284851 , -1.2847253, -1.2849066, -1.2848204],
       ...,
       [-1.284877 , -1.2848783, -1.284853 , -1.284804 ],
       [-1.2848401, -1.2848557, -1.2847972, -1.2848665],
       [-1.2848748, -1.2848799, -1.2848252, -1.2848618]], dtype=float32), label_ids=array([1, 3, 1, ..., 1, 2, 2]), metrics={'test_loss': 1.386292576789856, 'test_accuracy': 0.25727773406766324, 'test_runtime': 16.0096, 'test_samples_per_second': 79.39, 'test_steps_per_second': 9.932})


Instruction: Pytorch - If you detach a nn.module in the middle of a network do all the modules prior to that one not get their gradient calculated?

Input: So let's say I have X an input and a sequential network of net A, net B and net C. If I detach net B and I put X through A->B->C, because B is detached do I lose gradient information from A? I would assume no? I'm assuming it would just treat B like a constant to be added to the output of A rather than something differentiable.

Output:
TLDR; Preventing gradient computation on B won't stop computing gradients for the upstream network A.

I think there is some confusion on what you consider "detaching a model". In my opinion, there are three things to keep in mind with this kind of thing:

You can detach a tensor which effectively detaches it from the computational graph, i.e. if this tensor is used to compute another tensor requiring gradient, the backpropagation step will not propagate past this "detached" tensor.

In your way of describing "detaching a model", you can disable gradient computation on given layers of your network by switching the requires_grad to False on its parameters. This can done in a single line at the module level with nn.Module.requires_grad_. So in your case doing B.requires_grad_(False) will freeze the parameters of B such that they can't be updated. In other words, the gradients of the parameters of B won't be computed however the intermediate gradients used to propagate to A will! Here is a minimal example:
>>> A = nn.Linear(10,10)
>>> B = nn.Linear(10,10)
>>> C = nn.Linear(10,10)

# disable gradient computation on B
>>> B.requires_grad_(False)

# dummy input, inference, and backpropagation
>>> x = torch.rand(1,10, requires_grad=True)
>>> C(B(A(x))).mean().backward()

We can now check that gradients of C and A have indeed be filled properly:
>>> A.weight.grad.sum()
tensor(0.3281)

>>> C.weight.grad.sum()
tensor(-1.6335)

However of course, B.weight.grad returns None.

Lastly, yet another behaviour is when using the no_grad context manager. This effectively kills the gradient. If you do something like:
>>> yA = A(x)
>>> with torch.no_grad():
...    yB = B(yA)
>>> yC = C(yB)

Here yC is already detached from the network.


Instruction: How to connect a LSTM layer to a Linear layer in Pytorch

Input:
I am doing a classification task of MFCC (time-series data) using LSTM.
I have input (16,60,40)  (Batch,step,features)
class model(nn.Module):
  def __init__(self,ninp,num_layers,class_num,nhid=128):
      super().__init__()

      self.lstm_nets = nn.LSTM(input_size=ninp,hidden_size=nhid,num_layers=num_layers,
      batch_first=True,dropout=0.2,bidirectional=False)
      self.FC = nn.Linear(nhid,class_num)
      self.tanh = nn.Tanh()
      self.softmax = nn.LogSoftmax(1)

  def forward(self,X):

      device = 'cuda:0'
      out, (ht, ct) = self.lstm_nets(X)

      # out = ht.contiguous().view(16,-1)

      out = self.tanh(out)

      out = self.FC(out)

      Out = self.softmax(out)
      return Out

model = model(ninp=X.shape[2],num_layers=1,class_num=32,nhid=128)
loss_function = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.5e-4)

If I have out = ht.contiguous().view(16,-1) that flattens the LSTM output, I got error
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-96-a7e2ba68dcd9> in <module>()
     11
     12         optimizer.zero_grad()
---> 13         y_pred = model(X)
     14         # calculate loss function
     15         loss = loss_function(y_pred, y)

3 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/linear.py in forward(self, input)
    101
    102     def forward(self, input: Tensor) -> Tensor:
--> 103         return F.linear(input, self.weight, self.bias)
    104
    105     def extra_repr(self) -> str:

RuntimeError: mat1 and mat2 shapes cannot be multiplied (16x32 and 128x32)

If I have out = out.contiguous().view(16,-1) that flattens the LSTM output, I got error RuntimeError: mat1 and mat2 shapes cannot be multiplied  (16x7680 and 128x32)
If I remove the Flatten Step, I got such an error. RuntimeError: Expected target size [16, 32], got [16]
In addition, I found examples online do not flatten the output of LSTM
Thanks for any help.

Output:
In each timestep of an LSTM the input goes through a simple neural network and the output gets passed to the next timestep.
The output out of function
out, (ht, ct) = self.lstm_nets(X)

contains a list of ALL outputs (i.e the output of the neural networks of every timestep). Yet, in classification, you mostly only really care about the LAST output. You can get it like this:
out = out[:, -1]

This output has now the shape (hidden-size, 1).
So in your case your forward function should look like this:
 def forward(self,X):

      device = 'cuda:0'
      out, (ht, ct) = self.lstm_nets(X)
      out = out[: ,-1]

      out = self.tanh(out)

      out = self.FC(out)

      Out = self.softmax(out)
      return Out


Instruction: how to do KFold cross validation with TXT files

Input:
for example I have 10  .txt  files, in order to divide in test and train data.
(test_rate = 0.2 which means 2 test data and 8 train data)
In that case, the whole KFold cross validation should run 45 times (C[10,2])
how to do this in python? using sklearn's KFold function(code below) or other methods.
Much thanks for your reply.
import pandas as pd
import numpy as np
from sklearn.model_selection import KFold
KFold(n_splits=2, random_state=None, shuffle=False)

Output:
Yes, you can use sklearn. You should use 5-Fold cross validation if you want your test data to be 0.2 of the whole dataset. Because in 5-Fold CV, you divide your data into 5 splits and use 4 of them for training, remaining 1 for testing, each time. So, n_splits should be 5.
fnames = np.array([
  "1.txt",
  "2.txt",
  "3.txt",
  "4.txt",
  "5.txt",
  "6.txt",
  "7.txt",
  "8.txt",
  "9.txt",
  "10.txt"
])
kfold = KFold(n_splits=5)

for i, (train_idx, test_idx) in enumerate(kfold.split(fnames)):
  print(f"Fold {i}")
  train_fold, test_fold = fnames[train_idx], fnames[test_idx]
  print(f"\tlen train fold: {len(train_fold)}")
  print(f"\tTrain fold: {train_fold}")
  print(f"\tlen test fold: {len(test_fold)}")
  print(f"\tTest fold: {test_fold}")

This prints
Fold 0
    len train fold: 8
    Train fold: ['3.txt' '4.txt' '5.txt' '6.txt' '7.txt' '8.txt' '9.txt' '10.txt']
    len test fold: 2
    Test fold: ['1.txt' '2.txt']
Fold 1
    len train fold: 8
    Train fold: ['1.txt' '2.txt' '5.txt' '6.txt' '7.txt' '8.txt' '9.txt' '10.txt']
    len test fold: 2
    Test fold: ['3.txt' '4.txt']
Fold 2
    len train fold: 8
    Train fold: ['1.txt' '2.txt' '3.txt' '4.txt' '7.txt' '8.txt' '9.txt' '10.txt']
    len test fold: 2
    Test fold: ['5.txt' '6.txt']
Fold 3
    len train fold: 8
    Train fold: ['1.txt' '2.txt' '3.txt' '4.txt' '5.txt' '6.txt' '9.txt' '10.txt']
    len test fold: 2
    Test fold: ['7.txt' '8.txt']
Fold 4
    len train fold: 8
    Train fold: ['1.txt' '2.txt' '3.txt' '4.txt' '5.txt' '6.txt' '7.txt' '8.txt']
    len test fold: 2
    Test fold: ['9.txt' '10.txt']

You may want to give shuffle=True and a random_state in KFold for reproducibility.


Instruction: "can't optimize a non-leaf Tensor" on torch Parameter

Input:
I'm getting "can't optimize a non-leaf Tensor" on this bit of code
self.W_ch1 = nn.Parameter(
            torch.rand(encoder_feature_dim, encoder_feature_dim), requires_grad=True
        ).to(self.device)
self.W_ch1_optimizer = torch.optim.Adam([self.W_ch1], lr=encoder_lr)

Don't know why it's happening that should be the leaf tensor, because it has no children connected to it. It's just a torch.rand inside a nn.Parameter variable. It throws the error at the initialization of self.w_ch1_optmizer

Output:
The reason why it throws an error is that torch.Tensor.cuda has the effect of creating a reference for transferring the data doing so by registering a new node in the graph. In other words your parameter module W_ch1 is no longer a leaf node since you already have this "computation" tree:
nn.Parameter -> cuda:parameter = W_ch1

You can compare the following two results:
>>> p = nn.Parameter(torch.rand(1)).cuda()
>>> p.is_leaf
False

What you need to be doing is first instantiate your modules, and define your optimizer(s). Only then can you transfer them to the desired device. Not before:
>>> p = nn.Parameter(torch.rand(1))
>>> optimizer = optim.Adam([p], lr=lr)

Then you can transfer everything:
>>> p.cuda()
>>> optimizer.cuda()


Instruction: Best way to convert Tensor from [24, 512, 768, 1] to [24, 512, 14, 14]

Input:
I had an incompatibility issue with PyTorch tensor shape. Hence, I need to convert one tensor from shape [24, 512, 768, 1] to [24, 512, 14, 14].
What is the best way while trying to preserve as much info as possible about the original tensor representation?

Output:
I think this is an optimal solution, regardless of the data preservation you want to include:
import tensorflow as tf

t = tf.constant([[[[1]]*768]*512]*24)
t = tf.reshape(tf.constant(t.numpy()[:, :, :196]), (24, 512, 14, 14))
print(t.shape)

Output:
(24, 512, 14, 14)


Instruction: extract blocks of columns (as seperated subarrays) indicated by 1D binary array

Input:
Based on a 1D binary mask, for example, np.array([0,0,0,1,1,1,0,0,1,1,0]), I would like to extract the columns of another array, indicated by the 1's in the binary mask, as as sub-arrays/separate blocks, like [9, 3.5, 7]) and [2.8, 9.1] (I am just making up the numbers to illustrate the point).
So far what I have (again just as a demo to illustrate what my goal is, not the data where this operation will be performed):
arr = torch.from_numpy(np.array([0,0,0,1,1,1,0,0,1,1,0]))
split_idx = torch.where(torch.diff(arr) == 1)[0]+1
torch.tensor_split(arr, split_idx.tolist())

The output is:
(tensor([0, 0, 0]),
 tensor([1, 1, 1]),
 tensor([0, 0]),
 tensor([1, 1]),
 tensor([0]))

What I would like to have in the end is:
(tensor([1, 1, 1]),
 tensor([1, 1]))

Do you know how to implement it, preferably in pytorch, but numpy functions are also fine. A million thanks in advance!!

Output:
You can construct your tensor of slice indices with your approach. Only thing is you were missing the indices for the position of the end of each slice. You can do something like:
>>> slices = arr.diff().abs().nonzero().flatten()+1
tensor([ 3,  6,  8, 10])

Then apply tensor_split and slice to only keep every other element:
 >>> torch.tensor_split(arr, slices)[1::2]
 (tensor([1, 1, 1]), tensor([1, 1]))


Instruction: Evaluate BERT Model param.requires_grad

Input:
I have a doubt regarding the evaluation on the test set of my bert model. During the eval part param.requires_grad is suppose to be True or False? indipendently if I did a full fine tuning during training or not. My model is in model.eval() mode but I want to be sure to not force nothing wrong in the Model() class when i call it for evaluation. Thanks !
  if freeze_bert == 'True':
        for param in self.bert.parameters():
            param.requires_grad = False
            #logging.info('freeze_bert: {}'.format(freeze_bert))
            #logging.info('param.requires_grad: {}'.format(param.requires_grad))
    if freeze_bert == 'False':
        for param in self.bert.parameters():
            param.requires_grad = True

Output:
If you freeze your model then the parameter of the corresponding modules must not be updated, i.e. they should not require gradient computation: requires_grad=False.
Note nn.Module also has a requires_grad_ method:
if freeze_bert == 'True':
    self.bert.requires_grad_(False)

elif freeze_bert == 'False:
    self.bert.requires_grad_(True)

Ideally freeze_bert would be a boolean and you would simply do:
self.bert.requires_grad_(not freeze_bert)


Instruction: Pytorch: How exactly dataloader get a batch from dataset?

Input:
I am trying to use pytorch to implement self-supervised contrastive learning. There is a phenomenon that I can't understand.
Here is my code of transformation to get two augmented views from original data:
class ContrastiveTransformations:
  def __init__(self, base_transforms, n_views=2):
      self.base_transforms = base_transforms
      self.n_views = n_views

  def __call__(self, x):
      return [self.base_transforms(x) for i in range(self.n_views)]
contrast_transforms = transforms.Compose(
    [
        transforms.RandomResizedCrop(size=96),
        transforms.ToTensor(),
    ]
)
data_set = CIFAR10(
    root='/home1/data',
    download=True,
    transform=ContrastiveTransformations(contrast_transforms, n_views=2),
)

As the definition of ContrastiveTransformations, the type of data in my dataset is a list containing two tensors [x_1, x_2]. In my understanding, the batch from the dataloader should have the form of [data_batch, label_batch], and each item in data_batch is [x_1, x_2]. However, in fact, the form of the batch is in this way: [[batch_x1, batch_x2], label_batch], which is much more convinient to calculate infoNCE loss. I wonder that how DataLoader implements the fetch of the batch.
I have checked the code of DataLoader in pytorch, it seems that dataloader fetches the data in this way:
class _MapDatasetFetcher(_BaseDatasetFetcher):
    def __init__(self, dataset, auto_collation, collate_fn, drop_last):
        super(_MapDatasetFetcher, self).__init__(dataset, auto_collation, collate_fn, drop_last)

    def fetch(self, possibly_batched_index):
        if self.auto_collation:
            data = [self.dataset[idx] for idx in possibly_batched_index]
        else:
            data = self.dataset[possibly_batched_index]
        return self.collate_fn(data)

However I still didn't figure out how the dataloader generates the batch of x1 and x2 separately.
I would be very thankful if someone could give me an explanation.

Output:
In order to convert the separate dataset batch elements to an assembled batch, PyTorch's data loaders use a collate function. This defines how the dataloader should assemble the different elements together to form a minibatch
You can define your own collate function and pass it to your data.DataLoader with the collate_fn argument. By default, the collate function used by dataloaders is default_collate defined in torch/utils/data/_utils/collate.py.
This is the behaviour of the default collate function as described in the header of the function:
# Example with a batch of `int`s:
>>> default_collate([0, 1, 2, 3])
tensor([0, 1, 2, 3])

# Example with a batch of `str`s:
>>> default_collate(['a', 'b', 'c'])
['a', 'b', 'c']

# Example with `Map` inside the batch:
>>> default_collate([{'A': 0, 'B': 1}, {'A': 100, 'B': 100}])
{'A': tensor([  0, 100]), 'B': tensor([  1, 100])}

# Example with `NamedTuple` inside the batch:
>>> Point = namedtuple('Point', ['x', 'y'])
>>> default_collate([Point(0, 0), Point(1, 1)])
Point(x=tensor([0, 1]), y=tensor([0, 1]))

# Example with `Tuple` inside the batch:
>>> default_collate([(0, 1), (2, 3)])
[tensor([0, 2]), tensor([1, 3])]

# Example with `List` inside the batch:
>>> default_collate([[0, 1], [2, 3]])
[tensor([0, 2]), tensor([1, 3])]


Instruction: generate 1D tensor as unique index of rows of an 2D tensor (keeping the order and the original index)

Input:
This question is an updated version of generate 1D tensor as unique index of rows of an 2D tensor
Let's say we transform a 2D tensor to a 1D tensor by giving each, different row a different index, from 0 to the number of rows - 1.
[[1,4],[1,3],[1,2]] -> [0,1,2]

But if there are same rows, we repeat the index, like this below, the "original" index is k-1 for the k-th row
[[1,4],[1,2],[1,2]] -> [0,1,1]

Also if there is no repeat for the row (like the third row below), its index should be its original index, which is k-1 for the k-th row (for example 2 for [1,4]).
[[1,3],[1,3],[1,4]] -> [0,0,2]

A longer example:
[[1,2],[4,3],[1,4],[1,4],[4,3],[1,2],[5,6],[7,8]] -> [0,1,2,2,1,0,6,7]

How to implement this on PyTorch?

Output:
See the non-vectorized solution from @Michael
 d = {}; torch.tensor([d.setdefault(tuple(i.tolist()), e) for e, i in enumerate(t4)])

Another non-vectorized solution is
t4_list = t4.tolist(); torch.tensor(list(map(lambda x: t4_list.index(x), t4)))


Instruction: PIL Image library load and save changing pixel values

Input:
I am currently working on an Image segmentation problem. As part of preprocessing, I'm trying to create mask values for 2 classes [0, 1]. While, saving the processed tensor and loading them back produces different mask values. My current guess is under the hood PIL normalizing pixel values.
If so how do I stop it from doing?
I have created below a simple example explaining the same.
tensor_img = torch.where(torch.Tensor(250,250,3) > 0, 1, 0)
img_arr = tensor_img.numpy().astype(np.uint8)
np.unique(img_arr, return_counts=True)

(array([0, 1], dtype=uint8), array([148148,  39352]))
img = Image.fromarray(img_arr)
img.save("tmp.jpg")

#read saved image
img = PIL.create("tmp.jpg")
tensor(img).unique(return_counts=True)

(tensor([0, 1], dtype=torch.uint8), tensor([62288,   212]))

Output:
For this simple case (only 2 classes), you need to work with png and not jpeg since jpeg is a lossy compression and png is lossless.
tensor_img = torch.where(torch.Tensor(250,250,3) > 0, 1, 0)
img_arr = tensor_img.numpy().astype(np.uint8)
np.unique(img_arr, return_counts=True)

(array([0, 1], dtype=uint8), array([159189,  28311]))
img = Image.fromarray(img_arr)
img.save("tmp.png")

#read saved image
img = np.array(Image.open("tmp.png"))
torch.tensor(img).unique(return_counts=True)

(tensor([0, 1], dtype=torch.uint8), tensor([159189,  28311]))
For more classes it is preferred to work with color map.


Instruction: How to create an "islands" style pytorch matrix

Input:
Probably a simple question, hopefully with a simple solution:
I am given a (sparse) 1D boolean tensor of size [1,N].
I would like to produce a 2D tensor our of it of size [N,N], containing islands which are induced by the 1D tensor. It will be the easiest to observe the following image example, where the upper is the 1D boolean tensor, and the matrix below represents the resulted matrix:

Output:
Given a mask input:
>>> x = torch.tensor([0,0,0,1,0,0,0,0,1,0,0])

You can retrieve the indices with torch.diff:
>>> index = x.nonzero()[:,0].diff(prepend=torch.zeros(1), append=torch.ones(1)*len(x))
tensor([3., 5., 3.])

Then use torch.block_diag to create the diagonal block matrix:
>>> torch.block_diag(*[torch.ones(i,i) for i in index.int()])
tensor([[1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
        [1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.],
        [0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.],
        [0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.],
        [0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.],
        [0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],
        [0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1.],
        [0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1.]])


Instruction: How to solve TypeError: can’t convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first

Input:
I am modifying the 'train model' function below so to plot loss and accuracy graphs at every epochs during traning
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    losses=[]
    accuracies=[]

    y_loss = {}  # loss history
    y_loss['aug1_train'] = []
    y_loss['valid'] = []
    y_acc = {}
    y_acc['aug1_train'] = []
    y_acc['valid'] = []

    x_epoch = []
    fig = plt.figure()
    ax0 = fig.add_subplot(121, title="loss")
    ax1 = fig.add_subplot(122, title="accuracy")

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['aug1_train', 'valid']:
            if phase == 'aug1_train':
                scheduler.step()
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels,paths in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'aug1_train'):
                    outputs = model(inputs)

                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'aug1_train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f} '.format(
                phase, epoch_loss, epoch_acc))

            y_loss[phase].append(epoch_loss)
            y_acc[phase].append(epoch_acc)

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
                def draw_curve(current_epoch):
                    x_epoch.append(current_epoch)
                    ax0.plot(x_epoch, y_loss['aug1_train'], 'bo-', label='train')
                    ax0.plot(x_epoch, y_loss['valid'], 'ro-', label='val')
                    ax1.plot(x_epoch, y_acc['aug1_train'], 'bo-', label='train')
                    ax1.plot(x_epoch, y_acc['valid'], 'ro-', label='val')

                    if current_epoch == 0:
                        ax0.legend()
                        ax1.legend()

                    fig.savefig(os.path.join('/content/drive/My Drive/Stanford40/Graphs', 'train.jpg'))

                draw_curve(epoch)

            if phase=='aug1_train':
              losses.append(epoch_loss)
              accuracies.append(epoch_acc)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model,losses,accuracies

and I load the Densenet161 for traning as below
#Load Pretrained Densenet161 model
model_ft = models.densenet161(pretrained=True)
model_ft.classifier=nn.Linear(2208,11)

model_ft = model_ft.to(device)

criterion = nn.CrossEntropyLoss()

# Observe that all parameters are being optimized
opt = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
sched = lr_scheduler.StepLR(opt, step_size=5, gamma=0.1)

Finally I run the code below to start training:
model_ft,losses,accuracies = train_model(model_ft, criterion,opt ,sched,num_epochs=30)

and got this error as in the picture below:

How can I modify the code to get away from this error by using tensor.cpu() ?

Output:
What if try to get item() here
running_corrects += torch.sum(preds == labels.data).item()

and remove double() when dividing?
epoch_acc = running_corrects / dataset_sizes[phase]


Instruction: What would be the returned values from torch.Torch.size() apart from height and width?

Input:
I'm using a module (SwimIR) for image super resolution based on PyTorch
One of the operations that is done is:
b, c, h, w = img_lq.size()

E = torch.zeros(b, c, h*scale_factor, w*scale_factor).type_as(img_lq)
W = torch.zeros_like(E)

I.e., getting the tensor shape and creating a new tensor of zeros of the same shape. I'm trying to understand what would be the two assigned variables b and c returned by the method size() apart from h and w that are height and width respectively.
Looking in PyTorch documentation, I haven't found anything that could give me a clue about what are those 2 variables.

Output:
The torch.Tensor.size function returns the shape of the tensor. As would accessing torch.Tensor.shape directly.
In other words the first line is simply unpacking the dimension sizes of img_lq as: b ("batch size"), c ("channels"), h ("height"), and w ("width"). Of course, this code makes the assumption that img_lq has exactly four dimensions.


Instruction: How can I remove "for loop iteration" by using torch tensor operator?

Input:
I want to get rid of "for loop iteration" by using Pytorch functions in my code. But the formula is complicated and I can't find a clue. Can the "for loop iteration" in the below replaced with the Torch operation?
B=10
L=20
H=5

mat_A=torch.randn(B,L,L,H)
mat_B=torch.randn(L,B,B,H)
tmp_B=torch.zeros_like(mat_B)

for x in range(L):
    for y in range(B):
        for z in range(B):
            tmp_B[:,y,z,:]+=mat_B[x,y,z,:]*mat_A[z,x,:,:]

Output:
This looks like a good setup for applying torch.einsum. However, we first need to explicit the : placeholders by defining each individual accumulation term.
In order to do so, consider the shape of your intermediate tensor results. The first, mat_B[x,y,z] is shaped (H,), while the second mat_A[z,x,] is shaped (L, H).
In pseudo-code your initial operation is as follows:
for x, y, z, l, h in LxBxBxLxH:
    tmp_B[:,y,z,:] += mat_B[x,y,z,:]*mat_A[z,x,:,:]

Knowing this, we can reformulate your initial loop in pseudo-code as:
for x, y, z, l, h in LxBxBxLxH:
    tmp_B[l,y,z,h] += mat_B[x,y,z,h]*mat_A[z,x,l,h]

Therefore, we can apply torch.einsum by using the same notation as above:
>>> torch.einsum('xyzh,zxlh->lyzh', mat_B, mat_A)


Instruction: Why is there extra paramaters in my simple CNN?

Input:
Here is my problem : when I look for the number of parameters in my first block, I see 36928 parameters (which is what I expect). But when I used this block to construct a model in an other class nn.Module, there are 1792 extra parameters and I can't figure out where they come from.
I put some code below to illustrate.
class Conv2dBlock(torch.nn.Module):
    def __init__(self, in_filters, out_filters, kernel_size=3):
        super(Conv2dBlock, self).__init__()
        self.conv2d_seq = torch.nn.Sequential()
        for k in range(2):
            self.conv2d_seq.append(torch.nn.Conv2d(in_channels=in_filters, out_channels=out_filters, kernel_size=kernel_size, padding='same'))
            self.conv2d_seq.append(torch.nn.ReLU())
            in_filters = out_filters

    def forward(self, input):
        out = self.conv2d_seq(input)
        return out

En then, I use this block in an other nn.Module :
class EncoderBlock(torch.nn.Module):
    def __init__(self):
        super(EncoderBlock, self).__init__()
        self.conv2d = Conv2dBlock(3, 64)
        self.maxpool = torch.nn.MaxPool2d(kernel_size=2)

    def forward(self, input):
        x = self.conv2d(input)
        p = self.maxpool(x)
        out = torch.nn.functional.dropout(p, 0.3)
        return x, out

And finaly :
class UNet_model(torch.nn.Module):
    def __init__(self):
        super(UNet_model, self).__init__()
        self.encoder_block1 = EncoderBlock()

    def forward(self, input):
        p1 = self.encoder_block1(input)
        # I removed useless code

        return p1

model = UNet_model()
summary(model, (3,128,128))

This last class constructs a model with 38 720 parameters, instead of 36 928. It seems there is an extra convolutional layer ((3,64, (3,3)) = 1792 params) applied twice to the input... I don't understand.
Can somebody take a look ?
Thanks !

Output:
First of all, torch.nn.Sequential() doesn't support append method, it should be changed to add_module, like this:
for k in range(2):
   self.conv2d_seq.add_module(f"conv_{k}",torch.nn.Conv2d(in_channels=in_filters, out_channels=out_filters, kernel_size=kernel_size, padding='same'))
   self.conv2d_seq.add_module(f"relu_{k}",torch.nn.ReLU())
   in_filters = out_filters

Second, if you run torchinfo summary on the initial block you will see:
==========================================================================================
Layer (type:depth-idx)                   Output Shape              Param #
==========================================================================================
Conv2dBlock                              [1, 64, 64, 64]           --
├─Sequential: 1-1                        [1, 64, 64, 64]           --
│    └─Conv2d: 2-1                       [1, 64, 64, 64]           1,792
│    └─ReLU: 2-2                         [1, 64, 64, 64]           --
│    └─Conv2d: 2-3                       [1, 64, 64, 64]           36,928
│    └─ReLU: 2-4                         [1, 64, 64, 64]           --
==========================================================================================
Total params: 38,720
Trainable params: 38,720
Non-trainable params: 0
Total mult-adds (M): 158.60
==========================================================================================
Input size (MB): 0.05
Forward/backward pass size (MB): 4.19
Params size (MB): 0.15
Estimated Total Size (MB): 4.40
==========================================================================================

So you can see that you have two conv layers  (1,792 + 36,928) as you specified 2 layers in your for loop: for k in range(2).


Instruction: unsupported layer type "ScatterNDUpdate"

Input:
This issue relates to IR inferencing using OpenVINO on NCS2 (MYRIAD). The ONNX representation of the model I am trying to convert comprises 8 ScatterNDupdate layers, which OpenVINO doesn't support on NCS2.

Is there any way to get rid of those layers without affecting the model's functionality?
Does OpenVINO provide any alternative layer which will not affect the inferences quantitatively or qualitatively?
For future references, is there any way to avoid of scatterNDupdate layers? (reason for the creation of these layers)

Link to IR and model
Details:
OpenVINO version: 2022.1
ONNX Opset: 11 (tried on 13, not useful for resolving the issue)

Output:
The ScatterNDUpdate layer is indeed unsupported for NCS2 since it is not listed in this Supported Layer here.
Your available option is to create a custom layer for VPU that could replace the ScatterNDUpdate functionality. To enable operations not supported by OpenVINO™ out of the box, you need a custom extension for Model Optimizer, a custom nGraph operation set, and a custom kernel for your target device
You may refer to this guide.


Instruction: Getting different results after converting a model to from pytorch to ONNX

Input:
I'm coverting a googlenet model form pytorch to onnx using the following code:
torch.onnx.export(model,               # model being run
                  input_batch,                         # model input (or a tuple for multiple inputs)
                  "google-net-onnx-test.onnx",   # where to save the model (can be a file or file-like object)
                  export_params=True,        # store the trained parameter weights inside the model file
                  opset_version=10,          # the ONNX version to export the model to
                  do_constant_folding=True,  # whether to execute constant folding for optimization
                  input_names = ['input'],   # the model's input names
                  output_names = ['output'], # the model's output names
                  dynamic_axes={'input' : {0 : 'batch_size'},    # variable length axes
                                'output' : {0 : 'batch_size'}})

When I run the model on pytorch for this image:

I get the right results:
Samoyed 0.9378381967544556
Pomeranian 0.00828344002366066
Great Pyrenees 0.005603068508207798
Arctic fox 0.005527767818421125
white wolf 0.004741032607853413

But when I do it with ONNX I get this:

The pre and pos processing code is different for each case, I It should be equivalent.
This is the complete code in Pytorch:
import torch
from PIL import Image
from torchvision import transforms

model = torch.hub.load('pytorch/vision:v0.10.0', 'googlenet', pretrained=True)
model.eval()

input_image = Image.open(filename)
preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
input_tensor = preprocess(input_image)
input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model

# move the input and model to GPU for speed if available
if torch.cuda.is_available():
    input_batch = input_batch.to('cuda')
    model.to('cuda')

with torch.no_grad():
    output = model(input_batch)
# Tensor of shape 1000, with confidence scores over Imagenet's 1000 classes
#print(output[0])
# The output has unnormalized scores. To get probabilities, you can run a softmax on it.
probabilities = torch.nn.functional.softmax(output[0], dim=0)
print(probabilities[:2])

# Read the categories
with open("imagenet_classes.txt", "r") as f:
    categories = [s.strip() for s in f.readlines()]
# Show top categories per image
top5_prob, top5_catid = torch.topk(probabilities, 5)
for i in range(top5_prob.size(0)):
    print(categories[top5_catid[i]], top5_prob[i].item())

And this the code for ONNX
from PIL import Image
import imageio
import onnxruntime as ort
import numpy as np

import matplotlib.pyplot as plt
import numpy as np
from collections import namedtuple
import os
import time

def get_image(path):
    '''
        Using path to image, return the RGB load image
    '''
    img = imageio.imread(path, pilmode='RGB')
    return img

# Pre-processing function for ImageNet models using numpy
def preprocess(img):
    '''
    Preprocessing required on the images for inference with mxnet gluon
    The function takes loaded image and returns processed tensor
    '''
    img = np.array(Image.fromarray(img).resize((224, 224))).astype(np.float32)
    img[:, :, 0] -= 123.68
    img[:, :, 1] -= 116.779
    img[:, :, 2] -= 103.939
    img[:,:,[0,1,2]] = img[:,:,[2,1,0]]
    img = img.transpose((2, 0, 1))
    img = np.expand_dims(img, axis=0)

    return img

def predict(path):
    img_batch = preprocess(get_image(path))

    outputs = ort_session.run(
        None,
        {"input": img_batch.astype(np.float32)},
    )

    a = np.argsort(-outputs[0].flatten())
    results = {}
    for i in a[0:5]:
        results[labels[i]]=float(outputs[0][0][i])
    return results

ort_session = ort.InferenceSession("/content/google-net-onnx-test.onnx")

with open('synset.txt', 'r') as f:
    labels = [l.rstrip() for l in f]

image_path = "/content/dog.jpg"
predict(image_path)

I took the code of Pytorch from this tutorial
And the code for ONNX fro the github's ONNX Zoo
Edit:
From the comments of @jhso, I think the normalisation step:
mean=[0.485, 0.456, 0.406]
I seems to me that is equivalent to:
img[:, :, 0] -= 123.68
img[:, :, 1] -= 116.779
img[:, :, 2] -= 103.939

because:
constant = 256
a,b,c =  123.68/constant, 116.779/constant, 103.939/constant

print (f'{a:.3f} {b:.3f} {c:.3f}')
0.483 0.456 0.406

Regarding the std part, I'm not sure were it happend or if it is equivalent to:
img[:,:,[0,1,2]] = img[:,:,[2,1,0]]
img = img.transpose((2, 0, 1))

Also I ran the code again today and got a closer result:

Output:
Your preprocessing is wrong. Note that you have a center crop (less important) and a std deviation normalisation step you're not using.  You're also seemingly converting from BGR which isn't required when using PIL (it's more of an opencv thing) - happy to be corrected if I'm wrong as I'm going from memory.
preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

Your preprocessing stage should look something (ymmv) like this:
# Pre-processing function for ImageNet models using numpy
def preprocess(img):
    '''
    Preprocessing required on the images for inference with mxnet gluon
    The function takes loaded image and returns processed tensor
    '''
    img = np.array(Image.fromarray(img).resize((256, 256))).astype(np.float32)
    #center crop
    rm_pad = (256-224)//2
    img = img[rm_pad:-rm_pad,rm_pad:-rm_pad]
    #normalize to 0-1
    img /= 255.
    #normalize by mean + std
    img = (img - np.array([0.485, 0.456, 0.406]))/np.array([0.229, 0.224, 0.225])
    # img[:,:,[0,1,2]] = img[:,:,[2,1,0]] #don't think this is needed?
    img = img.transpose((2, 0, 1))
    img = np.expand_dims(img, axis=0)

    return img


Instruction: Why I'm getting error: "Boolean value of Tensor with more than one value is ambiguous"

Input:
I'm trying to run the following code:
l = torch.tensor([0, 1, 1, 1], requires_grad=False)
r = torch.rand(4, 2)

torch.nn.CrossEntropyLoss(r, l)

And I'm getting error:
RuntimeError: Boolean value of Tensor with more than one value is ambiguous

I looked here: Bool value of Tensor with more than one value is ambiguous in Pytorch but didn't understand the answers.
What do I need to change in order to run the code ?

Output:
The object you are manipulating, torch.nn.CrossEntropyLoss, is a PyTorch module class, not a function.
Therefore, you should either intialize it beforehand:
>>> ce_loss = nn.CrossEntropyLoss()
>>> cel_loss(r, l)

Or use the functional interface, i.e. torch.nn.functional.cross_entropy:
>>> F.cross_entropy(r, l)


Instruction: How to create an iterable DataPipe with PyTorch using txt files

Input:
I have two text files to train a transformer model. However, instead of using PyTorch's own datasets, I'm using something I downloaded from the internet.
source = open('./train_de.de', encoding='utf-8').read().split('\n')
target = open('./train_en.en', encoding='utf-8').read().split('\n')

With the code above, I have some Danish sentences in a list named "source", and their translation in English sentences in another list named "target".
My question is, how can I make an iterable DataPipe with PyTorch such that when I write something like:
source, target = next(iter(train_iter))

this will give me the Danish sentence with it's corresponding English translation in seperate strings?

Output:
You can use the Dataset and DataLoader class for that.
import torch

class YourDataset(torch.utils.data.Dataset):
    def __init__(self) -> None:
        self.source = open('./train_de.de', encoding='utf-8').read().split('\n')
        self.target = open('./train_en.en', encoding='utf-8').read().split('\n')

    def __getitem__(self, idx) -> torch.Tensor:
        # load one sample by index, e.g like this:
        source_sample = self.source[idx]
        target_sample = self.target[idx]

        # do some preprocessing, convert to tensor and what not

        return source_sample, target_sample

    def __len__(self):
        return len(self.source)

Now you can create a DataLoader from your custom dataset:
yourDataset = YourDataset()
dataloader = torch.utils.data.DataLoader(
        yourDataset,
        batch_size=8,
        num_workers=0,
        shuffle=True
)

Now you can iterate through the dataloader (using a loop or next(iter(...))) and one iteration returns as many samples as your batchsize (in this case 8 stacked samples).


Instruction: Are several subsequent `reshape` operations equivalent to only one?

Input:
In Pytorch,
I recently stumbled onto code that looks like this.
# initially, my_tensor is one_dimensional, of length b*x*y
my_tensor = my_tensor.reshape(b, x, y)
my_tensor = my_tensor.reshape(b, x*y)

Is it equivalent to only writing the second line?
my_tensor = my_tensor.reshape(b, x*y)

And in general, is doing several reshape operations always equivalent to only doing the last one?
Intuitively, I think so, but the documentation for reshape doesn’t really mention any invariant, and I couldn’t find information for the inner representation of tensors and how reshape changed that

Output: The reshape operation does not (need to) touch the underlying data. It simply adjusts the "meta-data" about the dimensions. So a series of reshape operations (without any operations in between!!) is equal to a single reshape operation.


Instruction: Inheriting the PyTorch DataLoader class

Input:
Can we inherit the DataLoader class? If so, are there any specific restrictions to it?
I know we can do so for the Dataset class, but I need to know specifically about the DataLoader.
I have some requirements for a specific use case which involves overriding some of the DataLoader methods and adding some new ones.
Would really appreciate the help!
Thanks....

Output:
Thanks @Jan and @TheodorPeifer for your useful inputs..! :)
I was able to resolve this with a combination of NamedTuple returns from the dataset __get_item__ calls with the "worker_specific_info"(torch.utils.data.get_worker_info()) along with a custom collate_fn implementation.


Instruction: Updating pytorch tensor by matching multiple boolean conditions on the original values

Input:
I currently am updating values in a Pytorch tensor using multiple OR conditions:
>>> import torch

>>> my_tensor = torch.tensor([0, 1, 2, 3, 4, 5])

>>> condition = ((my_tensor==1) | (my_tensor==4) | (my_tensor==5))
>>> my_tensor[condition] = 0

>>> my_tensor
[0, 0, 2, 3, 0, 0]

My list of conditions is much longer than the toy example above. Can the condition operator match a list? If not, what is the best solution?

Output:
How about using a combination of torch.where and  torch.isin like below:
>>> torch.where(torch.isin(my_tensor, torch.tensor([1,4,5])), 0, my_tensor)
tensor([0, 0, 2, 3, 0, 0])

Update, Second approach: We can use torch.reshape.
(torch.isin not exist in pytorch==1.9.1 as you say in comment)
>>> mask = (my_tensor == torch.reshape(torch.tensor([1,4,5]), (-1,1))).any(0)
>>> torch.where(mask, 0, my_tensor)


Instruction: Scipy Optimize Minimize: Optimization terminated successfully but not iterating at all

Input:
I am trying to code an optimizer finding the optimal constant parameters so as to minimize the MSE between an array y and a generic function over X. The generic function is given in pre-order, so for example if the function over X is x1 + c*x2 the function would be [+, x1, *, c, x2]. The objective in the previous example, would be minimizing:

sum_for_all_x  (y - (x1 + c*x2))^2

I show next what I have done to solve the problem. Some things that sould be known are:

X and y are torch tensors.
constants is the list of values to be optimized.

    def loss(self, constants, X, y):

        stack = []   # Stack to save the partial results
        const = 0    # Index of constant to be used
        for idx in self.traversal[::-1]:   # Reverse the prefix notation
            if idx > Language.max_variables:   # If we are dealing with an operator
                function = Language.idx_to_token[idx]  # Get its associated function
                first_operand = stack.pop()    # Get first operand
                if function.arity == 1:   # If the arity of the operator is one (e.g sin)
                    stack.append(function.function(first_operand))   # Append result
                else:   # Same but if arity is 2
                    second_operand = stack.pop()  # Need a second operand
                    stack.append(function.function(first_operand, second_operand))

            elif idx == 0:  # If it is a constant -> idx 0 indicates a constant
                stack.append(constants[const]*torch.ones(X.shape[0]))  # Append constant
                const += 1   # Update
            else:
                stack.append(X[:, idx - 1])   # Else append the associated column of X

        prediction = stack[0]
        return (y - prediction).pow(2).mean().cpu().numpy()

    def optimize_constants(self, X, y):
        '''
        # This function optimizes the constants of the expression tree.
        '''
        if 0 not in self.traversal:  # If there are no constants to be optimized return
            return self.traversal

        x0 = [0 for i in range(len(self.constants))]  # Initial guess
        ini = time.time()
        res = minimize(self.loss, x0, args=(X, y), method='BFGS', options={'disp': True})
        print(res)
        print('Time:', time.time() - ini)

The problem is that the optimizer theoretically terminates successfully but does not iterate at all. The output res would be something like that:
Optimization terminated successfully.
         Current function value: 2.920725
         Iterations: 0
         Function evaluations: 2
         Gradient evaluations: 1
      fun: 2.9207253456115723
 hess_inv: array([[1]])
      jac: array([0.])
  message: 'Optimization terminated successfully.'
     nfev: 2
      nit: 0
     njev: 1
   status: 0
  success: True
        x: array([0.])

So far I have tried to:

Change the method in the minimizer (e.g Nelder-Mead, SLSQP,...) but it happens the same with all of them.
Change the way I return  the result (e.g (y - prediction).pow(2).mean().item())

Output: It seems that scipy optimize minimize does not work well with Pytorch. Changing the code to use numpy ndarrays solved the problem.


Instruction: why my pytorch model summy has no parameters?

Input:
im beginer of pytorch.
I want to auto encoder model similar to U-Net.
so I make below code, and see summary using pytorch_model_summary but the result told me model have any parameters...
why my model have any parameters??
class unet_like(nn.Module):
    def __init__(self):
      super(unet_like, self).__init__()

    def conv2d_block(self, in_channels, out_channels, x):
      x = nn.Conv2d(in_channels=in_channels, out_channels = out_channels, kernel_size = 3, padding = "same")(x)
      x = nn.BatchNorm2d(out_channels)(x)
      x = nn.ReLU()(x)
      x = nn.Conv2d(in_channels = out_channels, out_channels = out_channels, kernel_size = 3, padding = "same")(x)
      x = nn.BatchNorm2d(out_channels)(x)
      x = nn.ReLU()(x)
      return x
    def forward(self, x):

      c1 = self.conv2d_block(3, 16, x)
      p1 = nn.MaxPool2d(kernel_size = 2)(c1)
      p1 = nn.Dropout2d(0.1)(p1)

      c2 = self.conv2d_block(16, 32, p1)
      p2 = nn.MaxPool2d(kernel_size = 2)(c2)
      p2 = nn.Dropout(0.1)(p2)

      c3 = self.conv2d_block(32, 64, p2)
      p3 = nn.MaxPool2d(kernel_size = 2)(c3)
      p3 = nn.Dropout(0.1)(p3)

      c4 = self.conv2d_block(64, 128, p3)
      p4 = nn.MaxPool2d(kernel_size = 2)(c4)
      p4 = nn.Dropout(0.1)(p4)

      c5 = self.conv2d_block(128, 256, p4)
      # nn.ConvTranspose2d(in_channels = 16, out_channels = 64, kernel_size = 3, stride = 1, padding = (1, 1)),
      # nn.ReLU(),

      u6 = nn.ConvTranspose2d(in_channels=256, out_channels=128, kernel_size = 2, stride = 2, output_padding = (0,1))(c5)
      print(u6.shape)
      print(c4.shape)
      u6 = torch.cat([u6, c4], 1) # u6: 128, c4: 128
      print(u6.shape)
      u6 = nn.Dropout(0.1)(u6)
      c6 = self.conv2d_block(256, 128, u6)

      u7 = nn.ConvTranspose2d(in_channels = 128, out_channels = 64, kernel_size = 2, stride = 2, output_padding = (1,0))(c6)
      u7 = torch.cat([u7, c3], 1)
      u7 = nn.Dropout(0.1)(u7)
      c7 = self.conv2d_block(128, 64, u7)

      u8 = nn.ConvTranspose2d(in_channels = 64, out_channels = 32, kernel_size = 2, stride = 2, output_padding = (0,1))(c7)
      u8 = torch.cat([u8, c2], 1)
      u8 = nn.Dropout(0.1)(u8)
      c8 = self.conv2d_block(64, 32, u8)

      u9 = nn.ConvTranspose2d(in_channels = 32, out_channels = 16, kernel_size = 2, stride = 2, output_padding = (0,1))(c8)
      u9 = torch.cat([u9, c1], 1)
      u9 = nn.Dropout(0.1)(u9)
      c9 = self.conv2d_block(32, 16, u9)
  #           in_channels, kernel_size,
  # outputs = Conv2D(1, (1, 1), activation = "sigmoid")(c9)

      c9 = nn.Conv2d(in_channels=16, out_channels = 1, kernel_size = 3, padding = (1,1))(c9)
      outputs = nn.Sigmoid()(c9)
      return outputs

model = unet_like().to("cpu")
print(pytorch_model_summary.summary(model, torch.tensor(train_images[:1], dtype = torch.float32).to("cpu"), show_input=True))

torch.Size([1, 128, 12, 9])
torch.Size([1, 128, 12, 9])
torch.Size([1, 256, 12, 9])
-----------------------------------------------------------------------
      Layer (type)         Input Shape         Param #     Tr. Param #
=======================================================================
       unet_like-1     [1, 3, 100, 75]               0               0
=======================================================================
Total params: 0
Trainable params: 0
Non-trainable params: 0
-----------------------------------------------------------------------

Output:
Your model definition must go in the initializer of your module class: __init__, while the forward handles the inference logic of that module. The PyTorch layers you are using are module classes, they need to be initialized before being use for inference. Unlike frameworks like keras, PyTorch doesn't need a compilation step, the logic is defined declaratively.
Here's something to get you started:
class UNetlike(nn.Module):
    def __init__(self):
      super().__init__()

        self.c1 = nn.Sequential(self.conv2d_block(3, 16),
                                nn.MaxPool2d(kernel_size=2),
                                nn.Dropout2d(0.1))

        self.c2 = nn.Sequential(self.conv2d_block(16, 32),
                                nn.MaxPool2d(kernel_size=2),
                                nn.Dropout2d(0.1))

        self.c2 = nn.Sequential(self.conv2d_block(32, 64),
                                nn.MaxPool2d(kernel_size=2),
                                nn.Dropout2d(0.1))

        self.c4 = nn.Sequential(self.conv2d_block(64, 128),
                                nn.MaxPool2d(kernel_size=2),
                                nn.Dropout2d(0.1))

    def conv2d_block(self, in_channels, out_channels):
        return nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=out_channels,
                      kernel_size=3, padding="same"),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(),
            nn.Conv2d(in_channels=out_channels, out_channels=out_channels,
                      kernel_size=3, padding="same"),
            nn.BatchNorm2d(out_channels),
            nn.ReLU())

    def forward(self, x):
        p1 = self.c1(x)
        p2 = self.c2(p1)
        p3 = self.c3(p2)
        p4 = self.c3(p3)
        # so on and so forth


Instruction: Am getting the following TypeError: can't convert cuda:0 device type tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first

Input:
I have the error being displayed whilst trying to plot the graph...
I am sharing the code in the following link:
https://colab.research.google.com/drive/1nILxtGSSCmOKrcHg3-_SL0l2bsIwUM1p?usp=sharing
I think I'm missing 'tensor.cpu()' somewhere but I can't really pinpoint it.. Everything else works :/ Can anyone help please?

Output: The thing is that the result of torch.sum(...) is a tensor. Try to change it in the following way in the lines where you add to running_corrects/val_running_corrects: torch.sum(...).item() and then update code accordingly. There are probably other ways to do it but it should work. Here is very similar question by the way: https://stackoverflow.com/a/72704295/14787618


Instruction: mat1 and mat2 shapes cannot be multiplied (128x4 and 128x64)

Input:
Could not find out why the mat1 from the convolutional network is 128x4 and not 4x128. The following is the convolutional network used:
model = torch.nn.Sequential(
torch.nn.Conv2d(2,32,kernel_size=3,padding=1),
torch.nn.ReLU(),
torch.nn.MaxPool2d(2,2),

torch.nn.Conv2d(32,64,kernel_size=3,padding=1),
torch.nn.ReLU(),
torch.nn.MaxPool2d(2,2),

torch.nn.Conv2d(64,128,kernel_size=3,padding=1),
torch.nn.ReLU(),
torch.nn.MaxPool2d(2,2,padding=1),
torch.nn.Flatten(),

torch.nn.Linear(128, 64),
torch.nn.ReLU(),
torch.nn.Linear(64,4)
)

The model training code is as follows:
epochs = 1000
losses = [] #A
for i in range(epochs): #B
    game = Gridworld(size=size, mode='static') #C
    # state_ = game.board.render_np().reshape(1,l1) + np.random.rand(1,l1)/10.0 #D
    state_ = game.board.render_np() + np.random.rand(size,size)/10.0 #D
    state1 = torch.from_numpy(state_).float() #E
    print(state1.shape)
    status = 1 #F
    while(status == 1): #G
        qval = model(state1) #H
        qval_ = qval.data.numpy()
        if (random.random() < epsilon): #I
            action_ = np.random.randint(0,4)
        else:
            action_ = np.argmax(qval_)

        action = action_set[action_] #J
        game.makeMove(action) #K
        state2_ = game.board.render_np().reshape(1,l1) + np.random.rand(1,l1)/10.0
        state2 = torch.from_numpy(state2_).float() #L
        reward = game.reward()
        with torch.no_grad():
            newQ = model(state2.reshape(1,l1))
        maxQ = torch.max(newQ) #M
        if reward == -1: #N
            Y = reward + (gamma * maxQ)
        else:
            Y = reward
        Y = torch.Tensor([Y]).detach()
        X = qval.squeeze()[action_] #O
        loss = loss_fn(X, Y) #P
        print(i, loss.item())
        clear_output(wait=True)
        optimizer.zero_grad()
        loss.backward()
        losses.append(loss.item())
        optimizer.step()
        state1 = state2
        if reward != -1: #Q
            status = 0
    if epsilon > 0.1: #R
        epsilon -= (1/epochs)

The error log shown is:
torch.Size([2, 12, 12])
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-22-d2f43f09fd01> in <module>()
     74     status = 1 #F
     75     while(status == 1): #G
---> 76         qval = model(state1) #H
     77         qval_ = qval.data.numpy()
     78         if (random.random() < epsilon): #I

3 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/linear.py in forward(self, input)
    101
    102     def forward(self, input: Tensor) -> Tensor:
--> 103         return F.linear(input, self.weight, self.bias)
    104
    105     def extra_repr(self) -> str:

RuntimeError: mat1 and mat2 shapes cannot be multiplied (128x4 and 128x64)

mat1 should be the output of the convolutional network after it is flattened, and mat2 is the linear network following it.
Appreciate any help. Thanks!

Output:
Here are the output shapes for each layer
Conv2d(2,32,kernel_size=3,padding=1)   # 32x12x12
MaxPool2d(2,2)                         # 32x6x6
Conv2d(32,64,kernel_size=3,padding=1)  # 64x6x6
MaxPool2d(2,2)                         # 64x3x3
Conv2d(64,128,kernel_size=3,padding=1) # 128x3x3
MaxPool2d(2,2,padding=1)               # 128x2x2
Flatten()                              # 128x4

You'll need to change the kernel parameters and padding sizes if you wish to obtain an output of a given shape. This link might help in calculating the output shapes after each layer.
Another approach is that you could take a transpose of the flattened array and pass it into the Linear layers. You'll need to add the line in your forward function like below
import torch
import torch.nn as nn

class NN(nn.Module):
  def __init__(self):
      super(NN, self).__init__()

      self.layer1 = nn.Sequential(
          torch.nn.Conv2d(2,32,kernel_size=3,padding=1),
          torch.nn.ReLU(),
          torch.nn.MaxPool2d(2,2))

      self.layer2 = nn.Sequential(
          torch.nn.Conv2d(32,64,kernel_size=3,padding=1),
          torch.nn.ReLU(),
          torch.nn.MaxPool2d(2,2))

      self.layer3 = nn.Sequential(
          torch.nn.Conv2d(64,128,kernel_size=3,padding=1),
          torch.nn.ReLU(),
          torch.nn.MaxPool2d(2,2,padding=1))

      self.flattened_tensor = nn.Flatten()

      self.linear_layer = nn.Sequential(
          torch.nn.Linear(128, 64),
          torch.nn.ReLU(),
          torch.nn.Linear(64,4)
      )

  def forward(self, inp):
    conv_output = self.layer3(self.layer2(self.layer1(inp)))
    flattened_output = self.flattened_tensor(conv_output)

    transposed_matrix = torch.transpose(flattened_output, 0, 1)

    linear_output = self.linear_layer(transposed_matrix)
    return linear_output

model = NN()
output = model(arr)


Instruction: KL Divergence of two torch.distribution.Distribution objects

Input:
I'm trying to determine how to compute KL Divergence of two torch.distribution.Distribution objects. I couldn't find a function to do that so far. Here is what I've tried:
import torch as t
from torch import distributions as tdist
import torch.nn.functional as F

def kl_divergence(x: t.distributions.Distribution, y: t.distributions.Distribution):
    """Compute the KL divergence between two distributions."""
    return F.kl_div(x, y)

a = tdist.Normal(0, 1)
b = tdist.Normal(1, 1)

print(kl_divergence(a, b))  # TypeError: kl_div(): argument 'input' (position 1) must be Tensor, not Normal

Output: torch.nn.functional.kl_div is computing the KL-divergence loss. The KL-divergence between two distributions can be computed using torch.distributions.kl.kl_divergence.