Instruction: How do I blend together tensors in patterns other than 2x2?

Input:
I have created code that blends tensors together, by first blending them together into rows and then blending the rows together into the final output. It works well for 4 tensors in a 2x2 pattern, but it fails to do a 2x3 (6 tensors), 3x3 (9 tensors), 4x4 (16 tensors) pattern.

The tensors are in the form of (B x C x H x W), where B is batch size, C is channels, H is height, and W is width.

For both the tiles to rows (tile_overlay()), and rows to final image (row_overlay()), I create a base tensor that I add the tiles/rows to. I suspect the issue with my code lies either with how I get the base tensor's dimensions, how I track where to put the rows/tiles on the base tensor, or the issue with both of those things.

import torch
from PIL import Image
import torchvision.transforms as transforms

def preprocess(image_name, image_size):
    image = Image.open(image_name).convert('RGB')
    if type(image_size) is not tuple:
        image_size = tuple([int((float(image_size) / max(image.size))*x) for x in (image.height, image.width)])
    Loader = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor()])
    tensor = (Loader(image) * 256).unsqueeze(0)
    return tensor

def deprocess(output_tensor):
    output_tensor = output_tensor.clone().squeeze(0).cpu() / 256
    output_tensor.clamp_(0, 1)
    Image2PIL = transforms.ToPILImage()
    image = Image2PIL(output_tensor.cpu())
    return image

def prepare_tile(tile, overlap, side='both'):
    lin_mask_left = torch.linspace(0,1,overlap).repeat(tile.size(3),1).repeat(3,1,1).unsqueeze(0)
    lin_mask_right = torch.linspace(1,0,overlap).repeat(tile.size(3),1).repeat(3,1,1).unsqueeze(0)
    if side == 'both' or side == 'right':
        tile[:,:,:,overlap:] = tile[:,:,:,overlap:] * lin_mask_right
    if side == 'both' or side == 'left':
        tile[:,:,:,:overlap] = tile[:,:,:,:overlap] * lin_mask_left
    return tile

def overlay_tiles(tile_list, rows, overlap):
    c = 1
    f_tiles = []
    base_length = 0
    for i, tile in enumerate(tile_list):
        if c == 1:
             f_tile = prepare_tile(tile.clone(), overlap, side='right')
             if i + 1<= rows[1]:
                 base_length += tile.clone().size(3) - overlap
        elif c == rows[1]:
             f_tile = prepare_tile(tile.clone(), overlap, side='left')
             if i + 1<= rows[1]:
                 base_length += tile.size(3) - overlap
        elif c > 0 and c < rows[1]:
             f_tile = prepare_tile(tile.clone(), overlap, side='both')
             if i + 1<= rows[1]:
                 base_length += tile.size(3) - (overlap*2)
        f_tiles.append(f_tile)
        if c == rows[1]:
             c = 0
        c+=1

    base_length += overlap
    base_tensor = torch.zeros(3, tile_list[0].size(2), base_length).unsqueeze(0)
    row_list = []
    for row in range(rows[1]):
        row_list.append(base_tensor.clone())

    row_val, num_tiles = 0, 0
    l_max = tile_list[0].size(3)
    for y in range(rows[0]):
        for x in range(rows[1]):
            if num_tiles % rows[1] != 0:
                l_max += (f_tiles[num_tiles].size(3)-overlap)*x
                l_min = l_max - f_tiles[num_tiles].size(3)
                row_list[row_val][:, :, :, l_min:l_max] = row_list[row_val][:, :, :, l_min:l_max] + f_tiles[num_tiles]
            else:
                row_list[row_val][:, :, :, :f_tiles[num_tiles].size(3)] = f_tiles[num_tiles]
                l_max = tile_list[0].size(3)
            num_tiles+=1
        row_val+=1
    return row_list

def prepare_row(row_tensor, overlap, side='both'):
    lin_mask_top = torch.linspace(0,1,overlap).repeat(row_tensor.size(3),1).rot90(3).repeat(3,1,1).unsqueeze(0)
    lin_mask_bottom = torch.linspace(1,0,overlap).repeat(row_tensor.size(3),1).rot90(3).repeat(3,1,1).unsqueeze(0)
    if side == 'both' or side == 'top':
        row_tensor[:,:,:overlap,:] = row_tensor[:,:,:overlap,:]  * lin_mask_top
    if side == 'both' or side == 'bottom':
        row_tensor[:,:,overlap:,:] = row_tensor[:,:,overlap:,:] * lin_mask_bottom
    return row_tensor

def overlay_rows(row_list, rows, overlap):
    c = 1
    f_rows = []
    base_height = 0
    for i, row_tensor in enumerate(row_list):
        if c == 1:
             f_row = prepare_row(row_tensor.clone(), overlap, side='bottom')
             if i + 1<= rows[0]:
                 base_height += row_tensor.size(2) - overlap
        elif c == rows[1]:
             f_row = prepare_row(row_tensor.clone(), overlap, side='top')
             if i + 1<= rows[0]:
                 base_height += row_tensor.size(2) - overlap
        elif c > 0 and c < rows[0]:
             f_row = prepare_row(row_tensor.clone(), overlap, side='both')
             if i + 1<= rows[0]:
                 base_height += tile.size(2) - (overlap*2)
        f_rows.append(f_row)
        if c == rows[0]:
             c = 0
        c+=1

    base_height += overlap
    base_tensor = torch.zeros(3, base_height, row_list[0].size(3)).unsqueeze(0)

    num_rows = 0
    l_max = row_list[0].size(3)
    for y in range(rows[0]):
            if num_rows > 0:
                l_max += (f_rows[num_rows].size(2)-overlap)*y
                l_min = l_max - f_rows[num_rows].size(2)
                base_tensor[:, :, l_min:l_max, :] = base_tensor[:, :, l_min:l_max, :] + f_rows[num_rows]
            else:
                base_tensor[:, :, :f_rows[num_rows].size(2), :] = f_rows[num_rows]
                l_max = row_list[0].size(2)
            num_rows+=1
    return base_tensor

def rebuild_image(tensor_list, rows, overlap_hw):
    row_tensors = overlay_tiles(tensor_list, rows, overlap_hw[1])
    full_tensor = overlay_rows(row_tensors, rows, overlap_hw[0])
    return full_tensor

test_tensor_1 = preprocess('brad_pitt.jpg', (1080,1080))
test_tensor_2 = preprocess('starry_night_google.jpg', (1080,1080))

tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone()]
rows = [2, 2]
overlap = [540, 540]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_2x2.png')

tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(),]
rows = [3, 3]
overlap = [540, 540]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_3x3.png')

tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone()]
rows = [4, 4]
overlap = [540, 540]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_4x4.png')

Running the above code will result in this error message when trying to create the 3x3 output:

Traceback (most recent call last):
  File "t0.py", line 148, in <module>
    complete_tensor = rebuild_image(tensor_list, rows, overlap)
  File "t0.py", line 126, in rebuild_image
    row_tensors = overlay_tiles(tensor_list, rows, overlap_hw[1])
  File "t0.py", line 68, in overlay_tiles
    row_list[row_val][:, :, :, l_min:l_max] = row_list[row_val][:, :, :, l_min:l_max] + f_tiles[num_tiles]
RuntimeError: The size of tensor a (0) must match the size of tensor b (1080) at non-singleton dimension 3

This is an example of what the 2x2 output looks like:

And this is a visual diagram with two examples of what I am doing:

Output:
The code now works with some changes:

import torch
from PIL import Image
import torchvision.transforms as transforms

def preprocess(image_name, image_size):
    image = Image.open(image_name).convert('RGB')
    if type(image_size) is not tuple:
        image_size = tuple([int((float(image_size) / max(image.size))*x) for x in (image.height, image.width)])
    Loader = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor()])
    tensor = (Loader(image) * 256).unsqueeze(0)
    return tensor

def deprocess(output_tensor):
    output_tensor = output_tensor.clone().squeeze(0).cpu() / 256
    output_tensor.clamp_(0, 1)
    Image2PIL = transforms.ToPILImage()
    image = Image2PIL(output_tensor.cpu())
    return image

def prepare_tile(tile, overlap, side='both'):
    h, w = tile.size(2), tile.size(3)
    lin_mask_left = torch.linspace(0,1,overlap).repeat(h,1).repeat(3,1,1).unsqueeze(0)
    lin_mask_right = torch.linspace(1,0,overlap).repeat(h,1).repeat(3,1,1).unsqueeze(0)
    if side == 'both' or side == 'right':
        tile[:,:,:,w-overlap:] = tile[:,:,:,w-overlap:] * lin_mask_right
    if side == 'both' or side == 'left':
        tile[:,:,:,:overlap] = tile[:,:,:,:overlap] * lin_mask_left
    return tile

def calc_length(w, overlap, rows):
    count = 0
    l_max = w
    for y in range(rows[0]):
        for x in range(rows[1]):
            if count % rows[1] != 0:
                l_max += w-overlap
                l_min = l_max - w
            else:
                l_max = w
            count+=1
    return l_max

def overlay_tiles(tile_list, rows, overlap):
    c = 1
    f_tiles = []
    base_length = 0
    for i, tile in enumerate(tile_list):
        if c == 1:
             f_tile = prepare_tile(tile.clone(), overlap, side='right')
        elif c == rows[1]:
             f_tile = prepare_tile(tile.clone(), overlap, side='left')
        elif c > 0 and c < rows[1]:
             f_tile = prepare_tile(tile.clone(), overlap, side='both')
        f_tiles.append(f_tile)
        if c == rows[1]:
             c = 0
        c+=1

    w = tile_list[0].size(3)
    base_length = calc_length(w, overlap, rows)
    base_tensor = torch.zeros(3, tile_list[0].size(2), base_length).unsqueeze(0)

    row_list = []
    for row in range(rows[0]):
        row_list.append(base_tensor.clone())

    row_num, num_tiles = 0, 0
    l_max = w
    for y in range(rows[0]):
        for x in range(rows[1]):
            if num_tiles % rows[1] != 0:
                l_max += w-overlap
                l_min = l_max - w
                print(num_tiles, l_max, l_min)
                row_list[row_num][:, :, :, l_min:l_max] = row_list[row_num][:, :, :, l_min:l_max] + f_tiles[num_tiles]
            else:
                row_list[row_num][:, :, :, :w] = f_tiles[num_tiles]
                l_max = w
            num_tiles+=1
        row_num+=1
    return row_list

def prepare_row(row_tensor, overlap, side='both'):
    lin_mask_top = torch.linspace(0,1,overlap).repeat(row_tensor.size(3),1).rot90(3).repeat(3,1,1).unsqueeze(0)
    lin_mask_bottom = torch.linspace(1,0,overlap).repeat(row_tensor.size(3),1).rot90(3).repeat(3,1,1).unsqueeze(0)
    if side == 'both' or side == 'top':
        row_tensor[:,:,:overlap,:] = row_tensor[:,:,:overlap,:]  * lin_mask_top
    if side == 'both' or side == 'bottom':
        row_tensor[:,:,overlap:,:] = row_tensor[:,:,overlap:,:] * lin_mask_bottom
    return row_tensor

def calc_height(h, overlap, rows):
    num_rows = 0
    l_max = h
    for y in range(rows[0]):
            if num_rows > 0:
                l_max += (h-overlap)
                l_min = l_max - h
            else:
                l_max = h
            num_rows+=1
    return l_max

def overlay_rows(row_list, rows, overlap):
    c = 1
    f_rows = []
    base_height = 0
    for i, row_tensor in enumerate(row_list):
        if c == 1:
             f_row = prepare_row(row_tensor.clone(), overlap, side='bottom')
        elif c == rows[0]:
             f_row = prepare_row(row_tensor.clone(), overlap, side='top')
        elif c > 0 and c < rows[0]:
             f_row = prepare_row(row_tensor.clone(), overlap, side='both')
        f_rows.append(f_row)
        if c == rows[0]:
             c = 0
        c+=1

    h = row_list[0].size(2)
    base_height = calc_height(h, overlap, rows)
    base_tensor = torch.zeros(3, base_height, row_list[0].size(3)).unsqueeze(0)

    num_rows = 0
    l_max = row_list[0].size(3)
    for y in range(rows[0]):
            if num_rows > 0:
                l_max += (h-overlap)
                l_min = l_max - h
                base_tensor[:, :, l_min:l_max, :] = base_tensor[:, :, l_min:l_max, :] + f_rows[num_rows]
            else:
                base_tensor[:, :, :h, :] = f_rows[num_rows]
                l_max = h
            num_rows+=1
    return base_tensor

def rebuild_image(tensor_list, rows, overlap_hw):
    row_tensors = overlay_tiles(tensor_list, rows, overlap_hw[1])
    full_tensor = overlay_rows(row_tensors, rows, overlap_hw[0])
    return full_tensor

test_tensor_1 = preprocess('brad_pitt.jpg', (1080,720))
test_tensor_2 = preprocess('starry_night_google.jpg', (1080,720))

print("2x2 Test")
tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone()]
rows = [2, 2]
overlap = [540, 260]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_2x2.png')

print("3x3 Test")
tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(),test_tensor_1.clone(),]
rows = [3, 3]
overlap = [540, 540]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_3x3.png')

print("3x4 Test")
tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone()]
rows = [3, 4]
overlap = [540, 260]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_3x4.png')

print("4x3 Test")
tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone()]
rows = [4, 3]
overlap = [540, 260]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_4x3.png')

print("4x4 Test")
tensor_list = [test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone(), \
test_tensor_1.clone(), test_tensor_2.clone(), test_tensor_1.clone(), test_tensor_2.clone()]
rows = [4, 4]
overlap = [540, 260]
complete_tensor = rebuild_image(tensor_list, rows, overlap)

ft = deprocess(complete_tensor.clone())
ft.save('complete_tensor_4x4.png')


Instruction: How do I install Pytorch 1.3.1 with CUDA enabled

Input:
I have a conda environment on my Ubuntu 16.04 system.

When I install Pytorch using:

conda install pytorch

and I try and run the script I need, I get the error message:

raise AssertionError("Torch not compiled with CUDA enabled")

From looking at forums, I see that this is because I have installed Pytorch without CUDA support.

I then tried:

conda install -c pytorch torchvision cudatoolkit=10.1 pytorch

but now I get the error:

    from torch.utils.cpp_extension import BuildExtension, CUDAExtension
  File "/home/username/miniconda3/envs/super_resolution/lib/python3.6/site-packages/torch/__init__.py", line 81, in <module>
    from torch._C import *
ImportError: /lib64/libc.so.6: version `GLIBC_2.14' not found

So it seems that these two installs are installing different versions of Pytorch(?). The first one that seemed to work was Pytorch 1.3.1.

My question: How do I install Pytorch with CUDA enabled, but ensure it is version 1.3.1 so that it works with my system?

Output:
Given that your system is running Ubuntu 16.04, it comes with glibc installed. You can check your version by typing ldd --version.
Keep in mind that PyTorch is compiled on CentOS which runs glibc version 2.17.
Then check the CUDA version installed on your system nvcc --version
Then install PyTorch as follows e.g. if your cuda version is 9.2:
conda install pytorch torchvision cudatoolkit=9.2 -c pytorch
If you get the glibc version error, try installing an earlier version of PyTorch.
If neither of the above options work, then try installing PyTorch from sources.
If you would like to set a specific PyTorch version to install, please set it as <version_nr> in the below command:
conda install pytorch=<version_nr> torchvision cudatoolkit=9.2 -c pytorch


Instruction: pytorch acess of weights and biases from a spcecific neuron

Input:
e.g:

input_size = 784
hidden_sizes = [128, 64]
output_size = 10

# Build a feed-forward network
model = nn.Sequential(nn.Linear(input_size, hidden_sizes[0]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[0], hidden_sizes[1]),
                      nn.ReLU(),
                      nn.Linear(hidden_sizes[1], output_size),
                      nn.Softmax(dim=1))

i want to acess all the weights and the bias of the N-TH neuron in a specific layer. I know that model.layer[1].weight gives the acess to all weights in a layer, but i also want to know of what neuron this weight is.

Output:
Assume you have n neurons in the layer,
The weight should be in order from neuron[0] to neuron[n].
For example to access weights of a fully connected layer

Parameter containing:
tensor([[-7.3584e-03, -2.3753e-02, -2.2565e-02,  ...,  2.1965e-02,
      1.0699e-02, -2.8968e-02], #1st neuron weights
    [ 2.2930e-02, -2.4317e-02,  2.9939e-02,  ...,  1.1536e-02,
      1.9830e-02, -1.4294e-02], #2nd neuron weights
    [ 3.0891e-02,  2.5781e-02, -2.5248e-02,  ..., -1.5813e-02,
      6.1708e-03, -1.8673e-02], #3rd neuron weights
    ...,
    [-1.2596e-03, -1.2320e-05,  1.9106e-02,  ...,  2.1987e-02,
     -3.3817e-02, -9.4880e-03], #nth neuron weights
    [ 1.4234e-02,  2.1246e-02, -1.0369e-02,  ..., -1.2366e-02,
     -4.7024e-04, -2.5259e-02], #(n+1)th neuron weights
    [ 7.5356e-03,  3.4400e-02, -1.0673e-02,  ...,  2.8880e-02,
     -1.0365e-02, -1.2916e-02] #(n+2)th neuron weights], requires_grad=True)

For instance

[-7.3584e-03, -2.3753e-02, -2.2565e-02,  ...,  2.1965e-02,
          1.0699e-02, -2.8968e-02] will be all the weights of the 1st neuron

-7.3584e-03 is the weight to the 1st neuron in the next layer

-2.3753e-02 is the weight to the 2nd neuron in the next layer

-2.2565e-02 is the weight to the 3rd neuron in the next layer

[ 2.2930e-02, -2.4317e-02,  2.9939e-02,  ...,  1.1536e-02,
          1.9830e-02, -1.4294e-02] will be all the weights of the 2nd neuron

2.2930e-02 is the weight to the 1st neuron in the next layer

-2.4317e-02 is the weight to the 2nd neuron in the next layer

-2.2565e-02 is the weight to the 3rd neuron in the next layer


Instruction: How does one vectorize reinforcement learning environments?

Input: I have a Python class that conforms to OpenAI's environment API, but it's written in non-vectorized form i.e. it receives one input action per step and returns one reward per step. How do I vectorize the environment? I haven't been able to find any clear explanation on GitHub.

Output:
You could write a custom class that iterates over an internal tuple of environments while maintaining the basic Gym API. In practice, there will be some differences, because the underlying environments don't terminate on the same timestep. Consequently, it's easier to combine the standard step and reset functions in
one method called step. Here's an example:

class VectorEnv:
    def __init__(self, make_env_fn, n):
        self.envs = tuple(make_env_fn() for _ in range(n))

    # Call this only once at the beginning of training (optional):
    def seed(self, seeds):
        assert len(self.envs) == len(seeds)
        return tuple(env.seed(s) for env, s in zip(self.envs, seeds))

    # Call this only once at the beginning of training:
    def reset(self):
        return tuple(env.reset() for env in self.envs)

    # Call this on every timestep:
    def step(self, actions):
        assert len(self.envs) == len(actions)
        return_values = []
        for env, a in zip(self.envs, actions):
            observation, reward, done, info = env.step(a)
            if done:
                observation = env.reset()
            return_values.append((observation, reward, done, info))
        return tuple(return_values)

    # Call this at the end of training:
    def close(self):
        for env in self.envs:
            env.close()

Then you can just instantiate it like this:

import gym
make_env_fn = lambda: gym.make('CartPole-v0')
env = VectorEnv(make_env_fn, n=4)

You'll have to do a little bookkeeping for your agent to handle the tuple of return values when you call step. This is also why I prefer to pass a function make_env_fn to __init__, because it's easy to add wrappers like gym.wrappers.Monitor that track statistics for each environment individually and automatically.


Instruction: Obtain a set of embedding from pretrained model - vgg16 pytorch

Input:
For a certain project purpose I am trying to store the 1 * 4096 embeddings (The output right before the final layer) of around 6000 images into a pkl file. For the same, I am running an iteration over the 6000 images on vgg16 modified model in google colab. But it returns 'CUDA out of memory. Tried to allocate 14.00 MiB (GPU 0; 15.90 GiB total capacity; 14.86 GiB already allocated; 1.88 MiB free; 342.26 MiB cached)' error.

Whereas I have used the same dataset split into test-train for training and validating my model and that runs fine. I am wondering why obtaining and storing the embedding alone is becoming a heavy task in colab.
Is there any other way I can obtain the embeddings and store in a pkl file other than the below code.

embedding = []
vgg16 = vgg16.to(device)
for x in range (0, len(inputImages)) :
  input = transformations(inputImages[x]) //pre processing
  input = torch.unsqueeze(input, 0)
  input = input.to(device)
  embedding.append(vgg16(input))

The code is interupted at the last line with the CUDA out of memory error.

Output: The output that you have generated vgg16(input), thats still in cuda. This is so because this output is used for calculating the loss afterwards. So to avoid having your output being stored in CUDA and eat up your GPU memory, move it to CPU using .cpu().numpy(). If that throws an error, you might have to use .detach() as well to detach the variable.


Instruction: Max Pooling in VGG16 Before Global Average Pooling (GAP)?

Input:
I am currently using VGG16 with Global Average Pooling (GAP) before final classification layer. The VGG16 model used is the one provided by torchvision.

However, I noticed that before the GAP layer, there is a Max Pooling layer. Is this okay or should the Max Pooling layer be removed before the GAP layer? The network architecture can be seen below.

VGG(
  (features): Sequential(
    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (1): ReLU(inplace=True)
    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (3): ReLU(inplace=True)
    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (6): ReLU(inplace=True)
    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (8): ReLU(inplace=True)
    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (11): ReLU(inplace=True)
    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (13): ReLU(inplace=True)
    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (15): ReLU(inplace=True)
    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (18): ReLU(inplace=True)
    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (20): ReLU(inplace=True)
    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (22): ReLU(inplace=True)
    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (25): ReLU(inplace=True)
    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (27): ReLU(inplace=True)
    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    (29): ReLU(inplace=True)
    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  )
  (avgpool): AdaptiveAvgPool2d(output_size=1)   #GAP Layer
  (classifier): Sequential(
    (0): Linear(in_features=512, out_features=7, bias=True)
  )
)

Thanks in advance.

Output:
If you are going to train the classifier, it should be okay. Nonetheless, I wouldn't remove it either way.

It is worth mentioning that the max-pooling is part of the original architecture, as can be seen in Table 1 of the original paper: https://arxiv.org/pdf/1409.1556.pdf.


Instruction: CNN Pytorch Error : Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

Input:
I'm receiving the error,

  Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

Following is my code,

device    = torch.device('cuda:0')

trainData = torchvision.datasets.FashionMNIST('/content/', train=True, transform=None, target_transform=None, download=True)
testData  = torchvision.datasets.FashionMNIST('/content/', train=False, transform=None, target_transform=None, download=True)

class Net(nn.Module):
  def __init__(self):
    super().__init__()

    '''
    Network Structure:

    input >
    (1)Conv2D > (2)MaxPool2D >
    (3)Conv2D > (4)MaxPool2D >
    (5)Conv2D > (6)MaxPool2D >
    (7)Linear > (8)LinearOut

    '''

    # Creating the convulutional Layers
    self.conv1 = nn.Conv2d(in_channels=CHANNELS, out_channels=32, kernel_size=3)
    self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3)
    self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3)

    self.flatten = None
    # Creating a Random dummy sample to get the Flattened Dimensions
    x = torch.randn(CHANNELS, DIM, DIM).view(-1, CHANNELS, DIM, DIM)
    x = self.convs(x)

    # Creating the Linear Layers
    self.fc1   = nn.Linear(self.flatten, 512)
    self.fc2   = nn.Linear(512, CLASSES)

  def convs(self, x):

    # Creating the MaxPooling Layers
    x = F.max_pool2d(F.relu(self.conv1(x)), kernel_size=(2, 2))
    x = F.max_pool2d(F.relu(self.conv2(x)), kernel_size=(2, 2))
    x = F.max_pool2d(F.relu(self.conv3(x)), kernel_size=(2, 2))

    if not self.flatten:
      self.flatten = x[0].shape[0] * x[0].shape[1] * x[0].shape[2]
    return x

  # FORWARD PASS
  def forward(self, x):
    x = self.convs(x)
    x = x.view(-1, self.flatten)
    sm = F.relu(self.fc1(x))
    x = F.softmax(self.fc2(sm), dim=1)
    return x, sm

  x_train, y_train = training_set
  x_train, y_train = x_train.to(device), y_train.to(device)
  optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE)
  loss_func = nn.MSELoss()
  loss_log  = []

  for epoch in range(EPOCHS):
    for i in tqdm(range(0, len(x_train), BATCH_SIZE)):
        x_batch = x_train[i:i+BATCH_SIZE].view(-1, CHANNELS, DIM, DIM).to(device)
        y_batch = y_train[i:i+BATCH_SIZE].to(device)

        net.zero_grad()
        output, sm = net(x_batch)
        loss = loss_func(output, y_batch.float())
        loss.backward()
        optimizer.step()
    loss_log.append(loss)
    # print(f"Epoch : {epoch} || Loss : {loss}")

  return loss_log

train_set = (trainData.train_data, trainData.train_labels)
test_set  = (testData.test_data, testData.test_labels)

EPOCHS        = 5
LEARNING_RATE = 0.001
BATCH_SIZE    = 32

net = Net().to(device)

loss_log = train(net, train_set, EPOCHS, LEARNING_RATE, BATCH_SIZE)

And this is the Error that I'm getting,

RuntimeError                              Traceback (most recent call last)
<ipython-input-8-0db1a1b4e37d> in <module>()
      5 net = Net().to(device)
      6
----> 7 loss_log = train(net, train_set, EPOCHS, LEARNING_RATE, BATCH_SIZE)

6 frames
<ipython-input-6-7de4a78e3736> in train(net, training_set, EPOCHS, LEARNING_RATE, BATCH_SIZE)
     13
     14         net.zero_grad()
---> 15         output, sm = net(x_batch)
     16         loss = loss_func(output, y_batch.float())
     17         loss.backward()

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

<ipython-input-5-4fddc427892a> in forward(self, x)
     41   # FORWARD PASS
     42   def forward(self, x):
---> 43     x = self.convs(x)
     44     x = x.view(-1, self.flatten)
     45     sm = F.relu(self.fc1(x))

<ipython-input-5-4fddc427892a> in convs(self, x)
     31
     32     # Creating the MaxPooling Layers
---> 33     x = F.max_pool2d(F.relu(self.conv1(x)), kernel_size=(2, 2))
     34     x = F.max_pool2d(F.relu(self.conv2(x)), kernel_size=(2, 2))
     35     x = F.max_pool2d(F.relu(self.conv3(x)), kernel_size=(2, 2))

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    539             result = self._slow_forward(*input, **kwargs)
    540         else:
--> 541             result = self.forward(*input, **kwargs)
    542         for hook in self._forward_hooks.values():
    543             hook_result = hook(self, input, result)

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py in forward(self, input)
    343
    344     def forward(self, input):
--> 345         return self.conv2d_forward(input, self.weight)
    346
    347 class Conv3d(_ConvNd):

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py in conv2d_forward(self, input, weight)
    340                             _pair(0), self.dilation, self.groups)
    341         return F.conv2d(input, weight, self.bias, self.stride,
--> 342                         self.padding, self.dilation, self.groups)
    343
    344     def forward(self, input):

RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

I double-checked that my Neural Net and my Inputs both are in GPU. I'm still getting this error and I don't understand why!

Somebody, please help me to get out of this error.

Output: Cast your input x_batch to float. Use x_batch = x_batch.float() before you pass it through your model.


Instruction: Android(kotlin), how getting assets file path? (at pytorch mobile)

Input:
I'm trying PyTorch Mobile tutorial by kotlin. I want to load module, "model.pt" in assets file. But no idea to load module in assets file.

Java (written in PyTorch Mobile Tutorial "hello world")

Module module = Module.load(assetFilePath(this, "model.pt"));

kotlin

val module = Module.load("?????")

Output:
Declare this function:

fun assetFilePath(context: Context, asset: String): String {
    val file = File(context.filesDir, asset)

    try {
        val inpStream: InputStream = context.assets.open(asset)
        try {
            val outStream = FileOutputStream(file, false)
            val buffer = ByteArray(4 * 1024)
            var read: Int

            while (true) {
                read = inpStream.read(buffer)
                if (read == -1) {
                    break
                }
                outStream.write(buffer, 0, read)
            }
            outStream.flush()
        } catch (ex: Exception) {
            e.printStackTrace()
        }
        return file.absolutePath
    } catch (e: Exception) {
        e.printStackTrace()
    }
    return ""
}

And then use it as:

val module = Module.load(assetFilePath(this, "model.pt"))


Instruction: Why should the  (huggingface) Transformers library be installed on a virtual environment?

Input:
In the huggingface github it is written:

You should install  Transformers in a virtual environment. If you're
unfamiliar with Python virtual environments, check out the user guide.
Create a virtual environment with the version of Python you're going
to use and activate it.
Now, if you want to use  Transformers, you can install it with pip.
If you'd like to play with the examples, you must install it from
source.

Why should it be installed in a virtual python environment?
What are the advantages of doing that rather than installing it on python as is?

Output:
Summing up the comments in a community answer:

It's not needed to install huggingface Transformers in a virtual environment, it can be installed just like any other package though there are advantages of using a virtual environment, and is considered a good practice.

You want to work in virtual envs for all Python work you do, so that you don't interfere the system install of Python, and so that you don't have a big global list of hundreds of packages that have nothing to do with each other and that may have conflicting requirements.
Apart from that, using a virtual environment for your project also means that it is easily deployable to a different machine, because all the dependencies are self-contained and can be packaged up in one go.

More in this answer


Instruction: How do I pass a color image through a Pytorch convolutional layer with custom filter?

Input:
I'm trying to visualize what happens when a color image passes through a convolutional layer. For that, I'm setting custom weights with zeroes and ones. The problem I'm facing is that I'm losing the 3D channels, and get a 1D channel after passing the data through the layer.

import requests
from io import BytesIO
from PIL import Image
import torch
import torch.nn as nn
import matplotlib.pyplot as plt
import numpy as np

link = 'https://audimediacenter-a.akamaihd.net/system/production/media/85094/images' \
       '/2a4e98976b1f9088fe6ae883f2f29e4d8f3ed473/A1912967_x500.jpg?1575885688'

r = requests.get(link, timeout=10)
im = Image.open(BytesIO(r.content))

pic = np.array(im)

horizontal_filter = torch.zeros(5, 5)
horizontal_filter[2, :] = 1

print(horizontal_filter)

This is my custom filter:

tensor([[0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0.],
        [1., 1., 1., 1., 1.],
        [0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0.]])

Now I'm using the custom filter and repeat it to fit 3 channels.

hz = nn.Conv2d(in_channels=3,
               out_channels=3,
               kernel_size=5,
               stride=1,
               bias=None)

hz.weight.data =  horizontal_filter.type('torch.FloatTensor').repeat(1, 3, 1, 1)

print(hz.weight.data.shape)

This is the shape of the filter:

torch.Size([1, 3, 5, 5])

I pass it through the convolutional filter and I'm losing the 3 channels:

zz = hz(torch.tensor(pic[None, ...]).permute(0, 3, 1, 2).type('torch.FloatTensor'))

print(np.transpose(zz.detach().numpy(), (0, 2, 3, 1)).shape)

(1, 329, 496, 1)

If I plot it, I don't have the colors anymore.

z = np.transpose(zz.detach().numpy(), (0, 2, 3, 1))[0, :, :, 0]

f, axarr = plt.subplots()
axarr.imshow(z)
plt.show()

tl;dr: How do I pass a 3D picture through a convolutional layer and return an image with 3 channels?

Output:
The issue is that you are not repeating the channels enough. Since you have 3 input and output channels, the Conv weight matrix would be 3x3x5x5. Since you had set it to 1x3x5x5, it was able to output only 1 channel.

You need to make the following change

hz.weight.data =  horizontal_filter.type('torch.FloatTensor').repeat(3, 3, 1, 1)

Because of your filter, your output would have a max value of ~3700. So to view, divide by the max using z = z/np.max(z) and then you get


Instruction: IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

Input:
As I've read through some previous questions, I'm getting this error that probably has something to do with a discrepancy between dimensions of tensors but since this is my very first attempt of running PyTorch so I'm coming here because I have far to little intuition about it. Wanted to run a non-standard dataset (which I'm pretty sure I'm loading fine) on a basic MNIST setup to mess with it and see what moves what.

Traceback (most recent call last):

File "C:/Users/Administrator/Desktop/pytong/proj/pytorch_cnnv2.py", line 110, in <module>
loss = loss_func(output, b_y)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\nn\modules\module.py", line 547, in __call__
result = self.forward(*input, **kwargs)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\nn\modules\loss.py", line 916, in forward
ignore_index=self.ignore_index, reduction=self.reduction)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\nn\functional.py", line 1995, in cross_entropy
return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
File "C:\Users\Administrator\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\nn\functional.py", line 1316, in log_softmax
ret = input.log_softmax(dim)

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

The line making the fuss is the loss function:

loss = loss_func(output, b_y)

The rest of the code, sans imports and load:

class CNNModel(nn.Module):
def __init__(self):
    super(CNNModel, self).__init__()

    # Convolution 1
    self.cnn1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=5, stride=1, padding=0)
    self.relu1 = nn.ReLU()

    # Max pool 1
    self.maxpool1 = nn.MaxPool2d(kernel_size=2)

    # Convolution 2
    self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)
    self.relu2 = nn.ReLU()

    # Max pool 2
    self.maxpool2 = nn.MaxPool2d(kernel_size=2)

    # Fully connected
    self.fc1 = nn.Linear(338 * 4 * 4, 5)

def forward(self, x):
    # Convolution 1
    out = self.cnn1(x)
    print(out.shape)
    out = self.relu1(out)
    print(out.shape)

    # Max pool 1
    out = self.maxpool1(out)
    print(out.shape)

    # Convolution 2
    out = self.cnn2(out)
    print(out.shape)
    out = self.relu2(out)
    print(out.shape)

    # Max pool 2
    out = self.maxpool2(out)
    print('++++++++++++++ out')
    print(out.shape)
    # out = out.reshape(-1, 169 * 4 * 4)
    out = out.view(out.size(0), -1)
    print(out.shape)
    print('-----------------------')
    # Linear function (readout)
    out = self.fc1(out)
    print(out.shape)
    print('=======================')

    return out

if __name__ == '__main__':

print("Number of train samples: ", len(train_data))
print("Number of test samples: ", len(test_data))
print("Detected Classes are: ", train_data.class_to_idx) # classes are detected by folder structure

model = CNNModel()
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
loss_func = nn.CrossEntropyLoss()

# Training and Testing
for epoch in range(EPOCHS):
    print(enumerate(train_data_loader))
    for step, (x, y) in enumerate(train_data_loader):
        b_x = Variable(x)   # batch x (image)
        b_y = Variable(y)   # batch y (target)
        # print('============ b_x')
        # print(len(b_x))
        # print(b_x.data)
        # print('============ b_y')
        # print(len(b_y))
        # print(b_y.data)

        output = model(b_x)[0]
        loss = loss_func(output, b_y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if step % 50 == 0:
            test_x = Variable(test_data_loader)
            test_output, last_layer = model(test_x)
            pred_y = torch.max(test_output, 1)[1].data.squeeze()
            accuracy = sum(pred_y == test_y) / float(test_y.size(0))
            print('Epoch: ', epoch, '| train loss: %.4f' % loss.data[0], '| test accuracy: %.2f' % accuracy)

Plus the output of my prints that I've tried to use for diagnostics:

torch.Size([100, 16, 60, 60])
torch.Size([100, 16, 60, 60])
torch.Size([100, 16, 30, 30])
torch.Size([100, 32, 26, 26])
torch.Size([100, 32, 26, 26])
++++++++++++++ out
torch.Size([100, 32, 13, 13])
torch.Size([100, 5408])
-----------------------
torch.Size([100, 5])
=======================

Output:
The problem is in this line:

output = model(b_x)[0]

The [0] changes the shape from [100, 5] to [5], and the loss expects the other way. Just remove it:

output = model(b_x)


Instruction: How to find a good LR?

Input:
I'm very new to PyTorch and working on a project of mine to solve a Sudoku board. What I'm doing is giving the network a Tensor that has the board (9x9) and another 2 values, the first being the row and the second being the column.
My network:

class Network(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(9 * 9 + 2, 32)  # board + row and col
        self.fc2 = nn.Linear(32, 32)
        self.fc3 = nn.Linear(32, 9)  # the number in that spot

    def forward(self, x):
        x = f.relu(self.fc1(x))
        x = f.relu(self.fc2(x))
        x = self.fc3(x)
        return f.log_softmax(x, dim=1)

I have a for loop that iterates over each board, and then iterate on each block in the 9x9 grid and input the board with the row and column to the network.

optimizer = optim.Adam(net.parameters(), lr=0.1)
scheduler = lr_scheduler.CosineAnnealingLR(optimizer, len(quizzes), eta_min=0)
for i, board in enumerate(quizzes):
    new_board = [[val for val in row] for row in board]  # Don't affect original board
    for row in range(9):
        for col in range(9):
            if new_board[row][col] != 0:
                continue
            row_col[0] = row  # the row value
            row_col[1] = col  # the col value

            final_tensor = board_tensor.view(-1, 9 * 9 + 2)
            output = net(final_tensor)  # type: Tensor
            solution_num = solutions[i][row][col]
            solution_tensor = torch.tensor(solution_num - 1, dtype=torch.long).reshape(-1)  # do -1 because it needs to match the node.
            optimizer.zero_grad()
            loss = f.nll_loss(output, solution_tensor)
            loss.backward()
            optimizer.step()
            scheduler.step(epoch=epoch)
            new_board[row][col] = solution_num  # Add the new value into the board.
            avg_loss += loss.item()
            count += 1
    if i % 100 == 0:
        print(f"Loss: {round(avg_loss / count, 3)}. {i} / {len(quizzes)}. {epoch} / {EPOCHS}")
        avg_loss = 0
        count = 0

Now, my issue is that the network is just guessing the same number over and over. Every time I reset the network it guesses a different number, but after a bit it just stays constant and doesn't guess any other number. This of course makes the accuracy 11.111% (1/9) and I don't know how to get past this. I tried using MSELoss instead of NLL_loss but it didn't change any result, and switched between optim.Adam to optim.SGE. I am quite new to this whole topic so I don't know which functions I should be using (log_softmax, Adam, SGE, and all those types of functions for loss / optimization).

Does anyone know where I messed up? I tried changing the learning rate and adding a weight decay too, but that didn't help

Output:
I don't suspect your issue is with parameter tuning. I suspect that the model is unable to learn how to solve this problem in this way.

To start: given a single cell on an unsolved (but let's assume solvable) Sudoku board, it's likely that other cells need to be solved in order to be able to know the correct value for the current cell. Asking your network to implicitly solve the entire puzzle to write down the answer for a single cell doesn't really make sense. Further, I'm not sure this task is something ML can solve in the approach you're using.

If you really have your heart set on solving this using ML, add a tenth "I don't know the answer" class. Then alter your dataset to be aware of when a given cell is not yet knowable. This will still end up with a model that is limited to the skill level of the person creating the dataset though. This will also be a non-trivial amount of extra work and I think there are better ways for you to spend your time learning about ML.

Side note: Sudoku is a graph colouring problem. Or depending on how you want to deal with it, a constrained integer program. Kinda weird to use ML to solve a solved problem IMHO.


Instruction: How to save model architecture in PyTorch?

Input:
I know I can save a model by torch.save(model.state_dict(), FILE) or torch.save(model, FILE). But both of them don't save the architecture of model.

So how can we save the architecture of a model in PyTorch like creating a .pb file in Tensorflow ? I want to apply different tweaks to my model. Do I have any better way than copying the whole class definition every time and creating a new class if I can't save the architecture of a model?

Output:
You can refer to this article to understand how to save the classifier. To make a tweaks to a model, what you can do is create a new model which is a child of the existing model.

class newModel( oldModelClass):
    def __init__(self):
        super(newModel, self).__init__()

With this setup, newModel has all the layers as well as the forward function of oldModelClass. If you need to make tweaks, you can define new layers in the __init__ function and then write a new forward function to define it.


Instruction: CNN pytorch : How are parameters selected and flow between layers

Input:
I'm pretty new to CNN and have been following the below code. I'm not able to understand how and why have we selected the each argument of Conv2d() and nn.Linear ()  as they are i.e. the output, filter, channels, weights,padding and stride. I do understand the meaning of each though.  Can someone very succinctly explain the flow for each layer?  (Input Image Size is 32*32*3)

import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)
        self.pool = nn.MaxPool2d(2, 2)
        self.fc1 = nn.Linear(64 * 4 * 4, 500)
        self.fc2 = nn.Linear(500, 10)
        self.dropout = nn.Dropout(0.25)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        x = x.view(-1, 64 * 4 * 4)
        x = self.dropout(x)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x

Output:
I think you'll find receptive field arithmetics useful for your understanding.

Your net has 3 convolution layers each with kernel size of 3x3 and padding of 1 pixels, which means that the spatial output of your convolution layers is the same as their input.
Each conv layer is followed by a max pooling with stride 2, that is, it reduces the spatial dimensions by a factor of 2.

So, in the spatial domain, you have an input of size 32x32 after first conv and pool its dimensions are 16x16, after the second conv and pool it is 8x8 and after the third conv+pool it is 4x4.

As for the "feature"/"channel" dimension: the input has 3 channels. The first conv layer has 16 filters ("out_channels=16") then 32 and finally 64.
Thus, after three conv layers your feature map has 64 channels (per spatial location).
Overall, an input of size 3x32x32 becomes 64x4x4 after the three conv+pooling layers defined by your network.

a nn.Linear layer does not assign "spatial" meaning to its inputs and expects a 1D input (per entry in a minibatch), thus your forward function "eliminates" the spatial dimensions and converts x to a 1D vector using the x.view(-1, 64 * 4 * 4) command.


Instruction: Correct dimensions to convert a wide data frame to Pytorch tensor object?

Input:
I have this time-series  data frame that has 56 columns and 36508 samples; 55 are predictors and the last one is the output. I'm trying to fit a LSTM neural network and while I'll be able to fit the model, I'm having some hard time converting the features to Pytorch tensor objects. Currently I have already normalised the data between 0 and 1 and also split the data into train and test sets.

import torch
import torch.nn as nn

print(x_train.shape)
(27380, 55)

print(y_train.shape)
(27380,)

print(x_test.shape)
(9128, 55)

print(y_test.shape)
(9128,)

I've had no problems converting the target to a tensor object, since the series is only 1D like so:

y_train = torch.FloatTensor(y_train).view(-1)
print(y_train[:5])
tensor([0.7637, 0.6220, 0.6566, 0.6922, 0.6774])

But when it comes to converting the features, then I'm unable to figure out the dimensions that need to be specified. I've tried this:

x_train = torch.FloatTensor(x_train).view(-1, 55)

ValueError: could not determine the shape of object type 'DataFrame'

How do I properly convert the features dataset to a tensor object? Sadly the documentation seems vague.

Output:
Try converting to numpy and then to tensors:

x_train = torch.from_numpy(np.array(x_train).astype(np.float32))


Instruction: What is temperature in Self Attention Terminology?

Input:
I was reading the paper's code "Attention is All You Need" . Code linked at here.

I found this term called temperature. How is it related to the Q,K,V formula for Attention.

My understanding of Self Attention is

Attention = Softmax(matmul(Q,K.T),dim=-1)
Output = Attention.V

Looking for something to correct or augment my understanding.

Output:
In the paper they define "Scaled Dot-Product Attention" as :
Attention(Q, K, V) = matmul(softmax(matmul(Q,K.T) / sqrt(dk)), V)

where dk is the dimension of queries (Q) and keys(K)
In the implementation, temperature seems to be the square root of dk, as it's called from the init part of MultiHeadAttention class :
self.attention = ScaledDotProductAttention(temperature=d_k ** 0.5)

and it's used in ScaledDotProductAttention class which implements the formula above:
attn = torch.matmul(q / self.temperature, k.transpose(2, 3))

ScaledDotProductAttention class :
https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/fec78a687210851f055f792d45300d27cc60ae41/transformer/Modules.py#L7


Instruction: Multilingual Bert sentence vector captures language used more than meaning - working as interned?

Input:
Playing around with BERT, I downloaded the Huggingface Multilingual Bert and entered three sentences, saving their sentence vectors (the embedding of [CLS]), then translated them via Google Translate, passed them through the model and saved their sentence vectors.

I then compared the results using cosine similarity.

I was surprised to see that each sentence vector was pretty far from the one generated from the sentence translated from it (0.15-0.27 cosine distance) while different sentences from the same language were quite close indeed (0.02-0.04 cosine distance).

So instead of having sentences of similar meaning (but different languages) grouped together (in 768 dimensional space ;) ), dissimilar sentences of the same language are closer.

To my understanding the whole point of Multilingual Bert is inter-language transfer learning - for example training a model (say, and FC net) on representations in one language and having that model be readily used in other languages.

How can that work if sentences (of different languages) of the exact meaning are mapped to be more apart than dissimilar sentences of the same language?

My code:

import torch

import transformers
from transformers import AutoModel,AutoTokenizer

bert_name="bert-base-multilingual-cased"
tokenizer = AutoTokenizer.from_pretrained(bert_name)
MBERT = AutoModel.from_pretrained(bert_name)

#Some silly sentences
eng1='A cat jumped from the trees and startled the tourists'
e=tokenizer.encode(eng1, add_special_tokens=True)
ans_eng1=MBERT(torch.tensor([e]))

eng2='A small snake whispered secrets to large cats'
t=tokenizer.tokenize(eng2)
e=tokenizer.encode(eng2, add_special_tokens=True)
ans_eng2=MBERT(torch.tensor([e]))

eng3='A tiger sprinted from the bushes and frightened the guests'
e=tokenizer.encode(eng3, add_special_tokens=True)
ans_eng3=MBERT(torch.tensor([e]))

# Translated to Hebrew with Google Translate
heb1='חתול קפץ מהעץ והבהיל את התיירים'
e=tokenizer.encode(heb1, add_special_tokens=True)
ans_heb1=MBERT(torch.tensor([e]))

heb2='נחש קטן לחש סודות לחתולים גדולים'
e=tokenizer.encode(heb2, add_special_tokens=True)
ans_heb2=MBERT(torch.tensor([e]))

heb3='נמר רץ מהשיחים והפחיד את האורחים'
e=tokenizer.encode(heb3, add_special_tokens=True)
ans_heb3=MBERT(torch.tensor([e]))

from scipy import spatial
import numpy as np

# Compare Sentence Embeddings

result = spatial.distance.cosine(ans_eng1[1].data.numpy(), ans_heb1[1].data.numpy())

print ('Eng1-Heb1 - Translated sentences',result)

result = spatial.distance.cosine(ans_eng2[1].data.numpy(), ans_heb2[1].data.numpy())

print ('Eng2-Heb2 - Translated sentences',result)

result = spatial.distance.cosine(ans_eng3[1].data.numpy(), ans_heb3[1].data.numpy())

print ('Eng3-Heb3 - Translated sentences',result)

print ("\n---\n")

result = spatial.distance.cosine(ans_heb1[1].data.numpy(), ans_heb2[1].data.numpy())

print ('Heb1-Heb2 - Different sentences',result)

result = spatial.distance.cosine(ans_eng1[1].data.numpy(), ans_eng2[1].data.numpy())

print ('Heb1-Heb3 - Similiar sentences',result)

print ("\n---\n")

result = spatial.distance.cosine(ans_eng1[1].data.numpy(), ans_eng2[1].data.numpy())

print ('Eng1-Eng2 - Different sentences',result)

result = spatial.distance.cosine(ans_eng1[1].data.numpy(), ans_eng3[1].data.numpy())

print ('Eng1-Eng3 - Similiar sentences',result)

#Output:
"""
Eng1-Heb1 - Translated sentences 0.2074061632156372
Eng2-Heb2 - Translated sentences 0.15557605028152466
Eng3-Heb3 - Translated sentences 0.275478720664978

---

Heb1-Heb2 - Different sentences 0.044616520404815674
Heb1-Heb3 - Similar sentences 0.027982771396636963

---

Eng1-Eng2 - Different sentences 0.027982771396636963
Eng1-Eng3 - Similar sentences 0.024596810340881348
"""

P.S.

At least the Heb1 was closer to Heb3 than to Heb2.
This was also observed for the English equivalents, but less so.

Output:
The [CLS] Token somehow represents the input sequence, but how exactly is difficult to say. The language is of course an important characteristic of a sentence, probably more than meaning. BERT is a pretrained model which tries to model such characteristics as meaning, structure and also language. If you want to have a model, which helps you identify if two sentences of different language mean the same thing, I can think of two different approaches:

approach: You can train a classifier (SVM, logistic Regression or even some neuronal nets such as CNN) on that task.  Inputs: two [CLS]-Token, Output: Same meaning, or not same meaning.
As training data, you could choose [CLS]-Token-pairs of sentences of different language which are either of the same meaning  or not. To get meaningful results, you would need a lot of such sentence pairs. Luckily you can either generate them via google translate, or use a parallel texts such as the bible which exists in a lot of languages, and extract sentence pairs from there.
approach: Fine-tune the bert model on exactly that task:
As in the previous approach, you need a lot of training data.
A sample input to the BERT model would look like that:
A cat jumped from the trees and startled the tourists [SEP] חתול קפץ מהעץ והבהיל את התיירים

To classify if those sentences are of the same meaning, you would add a classification layer on top of the [CLS]-Token and Fine tune the whole Model on that task.

Note: I have never worked with a multilingual BERT-model, those approaches are what comes to my mind to accomplish the mentioned task. If you try those approaches, I would be interested to know how they perform .


Instruction: What is a buffer in Pytorch?

Input:
I understand what register_buffer does and the difference between register_buffer and register_parameters.

But what is the precise definition of a buffer in PyTorch?

Output:
This can be answered looking at the implementation:

def register_buffer(self, name, tensor):
    if '_buffers' not in self.__dict__:
        raise AttributeError(
            "cannot assign buffer before Module.__init__() call")
    elif not isinstance(name, torch._six.string_classes):
        raise TypeError("buffer name should be a string. "
                        "Got {}".format(torch.typename(name)))
    elif '.' in name:
        raise KeyError("buffer name can't contain \".\"")
    elif name == '':
        raise KeyError("buffer name can't be empty string \"\"")
    elif hasattr(self, name) and name not in self._buffers:
        raise KeyError("attribute '{}' already exists".format(name))
    elif tensor is not None and not isinstance(tensor, torch.Tensor):
        raise TypeError("cannot assign '{}' object to buffer '{}' "
                        "(torch Tensor or None required)"
                        .format(torch.typename(tensor), name))
    else:
        self._buffers[name] = tensor

That is, the buffer's name:

must be a string: not isinstance(name, torch._six.string_classes)
cannot contain a . (dot): '.' in name
cannot be an empty string: name == ''
cannot be an attribute of the Module: hasattr(self, name)
should be unique: name not in self._buffers

and the tensor (guess what?):

should be a Tensor: isinstance(tensor, torch.Tensor)

So, the buffer is just a tensor with these properties, registered in the _buffers attribute of a Module;


Instruction: Replace all indices in tensor within a range with 1s

Input:
def generate_mask(data : list, max_seq_len : int):
    """
    Generates a mask for data where each element is expected to be max_seq_len length after padding
    Args:
    data : The data being forwarded through LSTM after being converted to a tensor
    max_seq_len : The length of the names after being padded
    """
    batch_sz = len(data)
    ret = torch.zeros(1,batch_sz, max_seq_len, dtype=torch.bool)
    for i in range(batch_sz):
        name = data[i]

        for letter_idx in range(len(name)):
            ret[0][i][letter_idx] = 1

    return ret

I have this code for generating a mask and I really hate how I'm doing it. Essentially as you can see I'm just going through every name and turning each index from 0 to name length to 1, I'd prefer a more elegant way to do this.

Output:
Well, you can simplify to something like this:

# [...]
for i in range(batch_sz):
    ret[0, i, :len(data[i])] = 1


Instruction: Despite installing the torch vision pytorch library, I am getting an error saying that there is no module named torch vision

Input:
The error that I am getting when I use import torchvision is this:

Error Message

"*Traceback (most recent call last):
  File "/Users/gokulsrin/Desktop/torch_basics/data.py", line 4, in <module>
    import torchvision
ModuleNotFoundError: No module named 'torchvision'*"

I don't know what to do. I have tried changing the version of python from the native one to the one downloaded through anaconda. I am using anaconda as a package manager and have installed torch vision through anaconda as well as through pip commands.

Output:
From PyTorch installing Docs you should follow these steps:

In Anaconda use this command:
conda install pytorch torchvision cpuonly -c pytorch

In Pip use this command:
pip3 install torch==1.3.1+cpu torchvision==0.4.2+cpu -f https://download.pytorch.org/whl/torch_stable.html

Note: If you have an enabled CUDA card you can change the cpuonly option to cudatoolkit=10.1 or cudatoolkit=9.2
After successfully installing the package you can import it with the command import torchvision and the output should look like this:

Otherwise, there is something wrong when you are downloading the package from the Internet


Instruction: classification using 4-channel images in pytorch

Input: I have some gray scale and color images with label. I want to combine this gray and color images (4-channel) and run transfer learning using 4-channel images. How to do that?

Output:
If I understand the question correctly you want to combine 1 channel images and 3 channel images and get a 4 channel image and use this as your input.

If this is what you want to do you can just use torch.cat().

Some example code of loading two images and combining them along the channel dimension

import numpy as np
import torch
from PIL import Image

image_rgb = Image.open(path_to_rgb_image)
image_rgb_tensor = torch.from_numpy(np.array(image_rgb))
image_rgb.close()

image_grayscale = Image.open(path_to_grayscale_image))
image_grayscale_tensor = troch.from_numpy(np.array(image_grayscale))
image_grayscale.close()

image_input = torch.cat([image_rgb_tensor, image_grayscale_tensor], dim=2)

I assumed that the grayscale image you want to use translated to a tensor with the shape [..., ..., 1] and the rgb image to [..., ..., 3].


Instruction: How to use Pytorch Tensor object in Opencv without convert to numpy array?

Input:
I’m trying to develop a text detection application with Pytorch and Opencv in Python.
I can use Pytorch tensor with Opencv like below.

val = y[0,:,:,0].data.cpu().numpy()
cv2.threshold(val , 0.4, 1, 0)

But it takes a lot of time. I need to do this operation by using the tensor object. How can I do that?

Output:
Given that the last 0 in your threshold call means cv.THRESH_BINARY, it follows this function:

As your maxval is set to 1, you can replace this threshold call with something like this:

(y[0,:,:,0] > 0.4).float()

I am casting to float, but you can change that as you wish, ofc. Or even to something like:

(y[0,:,:,0] > 0.4).to(dtype=y.dtype)

so that it will remain with the same data type.


Instruction: Incorrect predictions on extracted images from text

Input:
I trained a model in PyTorch on the EMNIST data set - and got about 85% accuracy on the test set. Now, I have an image of handwritten text from which I have extracted individual letters, but I'm getting very poor accuracy on the images that I have extracted.

One hot mappings that I'm using -

letters_EMNIST = {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9',
         10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J',
         20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T',
         30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'd', 39: 'e',
         40: 'f', 41: 'g', 42: 'h', 43: 'n', 44: 'q', 45: 'r', 46: 't'}

For reference, this is an example of the image used for testing data -

And this is an example of the image I extracted -

How can I debug this?

Output:
First of all, you should check your extraction technique and whether it works correctly. Rest of the answer assumes this step is done.

Distribution of EMNIST and distribution of your extracted data is probably quite different, hence it might be hard to obtain good results.

There are some steps you may do to improve the score though.

Additional data

If you have some way to extract more images of letters and ciphers and label them appropriately you should use it during neural net training.

The more of those you get, the better your results probably be (provided data is of quite high quality, e.g. not many false positives).

Data augmentation

You could do this without too much work I think.

You have to remember though that data augmentation has to preserve labels. So no things like flipping (it would be fine for number 8 but u flipped could become n).

Augmentations which should be fine:

Small rotations (up to 20 degrees or so)
Small Gaussian noise or similar
CutOut with small patches (black rectangulars of size 3x3 pixels or similar being zeroed out on the image)
Gentle spatial transformations (rescale, shift, linear transformations)
MixUp (you mix linearly two images with different labels (e.g. image of A multiplied by 0.6 and cipher 2 multiplied by 0.4 and try to classify it as 0.6 A and 0.4 2). Remember labels dont have to be solely binary. This should help your network not to be overconfident with it's predictions

You can find all of those in albumentations third party library.

Model augmentation

For your model you could employ thing like dropout (be careful with it's integration with batch norm though), shake shake, Stochastic Depth etc.

Final

You can use all of those, remember to test how it performs. I tried to list them with the most promising approach on top.

One possibility would be to make the model more robust to variance via augmentation.


Instruction: When I run deep learning training code on Google Colab, do the resulting weights and biases get saved somewhere?

Input:
I am training some deep learning code from this repository on a Google Colab notebook. The training is ongoing and seems like it is going to take a day or two.

I am new to deep learning, but my question:

Once the Google Colab notebook has finished running the training script, does this mean that the resulting weights and biases will be hard written to a model somewhere (in the repository folder that I have on my Google Drive), and therefore I can then run the code on any test data I like at any point in the future? Or, once I close the Google Colab notebook, do I lose the weight and bias information and would have to run the training script again if I wanted to use the neural network?

I realise that this might depend on the details of the script (again, the repository is here), but I thought that there might be a general way that these things work also.

Any help in understanding would be greatly appreciated.

Output:
No; Colab comes with no built-in checkpointing; any saving must be done by the user - so unless the repository code does so, it's up to you.

Note that the repo would need to figure out how to connect to a remote server (or connect to your local device) for data transfer; skimming through its train.py, there's no such thing.

How to save model? See this SO; for a minimal version - the most common, and a reliable option is to "mount" your Google Drive onto Colab, and point save/load paths to direct

from google.colab import drive
drive.mount('/content/drive') # this should trigger an authentication prompt
%cd '/content/drive/My Drive/'
# alternatively, %cd '/content/drive/My Drive/my_folder/'

Once cd'd into, for example, DL Code in your My Drive (see below), you can simply do model.save("model0.h5"), and this will create model0.h5 in DL Code, containing entire model architecture & its optimizer. For just weights, use model.save_weights().


Instruction: How to add a Validation and Test Set in Pytorch Model?

Input:
I have Non Linear Regression Model ANN( X = [1000,3] , Y = [1000,8] ) with One hidden Layer(Nh = 6).

How to add a Validation(10% Dataset) and Test Set(10% Dataset) in this model ?

Model :

N, D_in, H, D_out = x.shape[0], x.shape[1], 6, y.shape[1]

model = nn.Sequential(OrderedDict([ ('fc1', nn.Linear(D_in, H)),
                                    #('Sig', nn.Sigmoid()),
                                    ('ISRU', ISRU()), # Add ISRU
                                    ('fc2', nn.Linear(H, D_out))]))

# Error -----
loss_fn = torch.nn.L1Loss(reduction='mean')

# Train -----
optimizer = torch.optim.Adam(model.parameters(), lr=1,eps=2**(-EPS))
epoch = 250
for t in range(epoch):
    # Forward pass: compute predicted y by passing x to the model.
    clear_output(wait=True)
    y_pred = model(X)

    # Compute and print loss.
    loss = loss_fn(y_pred, Y)
    if t % 100 == 99:
        print(t, loss.item())

    optimizer.zero_grad() ;
    loss.backward() ;
    optimizer.step() ;
if loss.item() < diff : lista = np.vstack((lista, [loss.item(),2,EPS])) ; diff = loss.item()

Output:
there are many ways to do this. you can use what @Shai suggested, I want to add what I would like to do. I ofen use train_test_split to split my data into train and test and then move forward to convert my train and test data to TensorDataset

if you like an easier solution, you can take a look at skorch it is a scikit learn wrapper for pytorch. I find it easy to use and more like the keras API, skorch will implement train test split automatically when you start training but you can also pass your custom train and test set to it.


Instruction: Understanding batch_size in CNNs

Input:
Say that I have a CNN model in Pytorch and 2 inputs of the following sizes:

input_1: [2, 1, 28, 28]
input_2: [10, 1, 28, 28]

Notes:

To reiterate, input_1 is batch_size == 2 and input_2 is batch_size == 10.
Input_2 is a superset of input_1. That is, input_2 contains the 2 images in input_1 in the same position.

My question is: how does the CNN process the images in both inputs? I.e. does the CNN process every image in the batch sequentially? Or does it concatenate all of the images in the batch size and then perform convolutions per usual?

The reason I ask is because:

The output of CNN(input_1) != CNN(input_2)[:2]

That is, the difference in batch_size results in slightly different CNN outputs for both inputs for the same positions.

Output:
CNN is a general term for convolutional neural networks. Depending on the particular architecture it may do different things. The main building blocks of CNNs are convolutions which do not cause any "crosstalk" between items in batch and pointwise nonlinearities like ReLU which do not either. However, most architectures also involve other operations, such as normalization layers - arguably the most popular is batch norm which does introduce crosstalk. Many models will also use dropout which behaves stochastically outside of eval mode (by default models are in train mode). Both above effects could lead to the observed outcome above, as well as other custom operations which could cause cross-talk across the batch.

Aside from that, because of numeric precision issues, your code may not give exactly the same results, even if it doesn't feature any cross-batch operations. This error is very minor but sufficient to manifest itself when checking with CNN(input_1) == CNN(input_2)[:2]. It is better to use allclose instead, with a suitable epsilon.


Instruction: Changing back to CPU with Google Colab

Input:
I am running a model on Google Colab.  The final step I would like to do is print an image, and show the top 5 classification predictions of the model.  Here is the code:

image = process_image(imgpath)

index = 17
plot = imshow(image, ax = plt)
plot.axis('off')
plot.title(cat_to_name[str(index)])
plot.show()

axes = predict(image, model)

yaxis = [cat_to_name[str(i)] for i in np.array(axes[1][0])]
y_pos = np.arange(len(yaxis))
xaxis = np.array(axes[0][0])

plt.barh(y_pos, xaxis)
plt.xlabel('probability')
plt.yticks(y_pos, yaxis)
plt.title('probability of flower classification')

plt.show()

I am getting this error when I run this cell:

TypeError                                 Traceback (most recent call last)
<ipython-input-19-d0bb6f461eec> in <module>()
     11 axes = predict(image, model)
     12
---> 13 yaxis = [cat_to_name[str(i)] for i in np.array(axes[1][0])]
     14 y_pos = np.arange(len(yaxis))
     15 xaxis = np.array(axes[0][0])

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in __array__(self, dtype)
    447     def __array__(self, dtype=None):
    448         if dtype is None:
--> 449             return self.numpy()
    450         else:
    451             return self.numpy().astype(dtype, copy=False)

TypeError: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

Is there a way to temporarily use CPU on Google Colab and in this particular step?  I don't really need to switch back to GPU because this is the final step in my code.

Output:
Try the following:

yaxis = [cat_to_name[str(i)] for i in axes[1][0].cpu()]
xaxis = axes[0][0].cpu().numpy()


Instruction: How to fit custom data into Pytorch DataLoader?

Input:
I have pre-processed and normalized my data, and split into training set and testing set. I have the following dimensions for my x_train and y_train:
Shape of X_Train:  (708, 256, 3)
Shape of Y_Train:  (708, 4)

As you can see, x_train is 3-D. How can I go about inputting it into the pytorch dataloader? What do I put for the class block?

class training_set(Dataset):
    def __init__(self,X,Y):

    def __len__(self):
        return

    def __getitem__(self, idx):
        return

training_set = torch.utils.data.TensorDataset(x_train, y_train)
train_loader = torch.utils.data.DataLoader(training_set, batch_size=50, shuffle=True)

Output:
x_train, y_train = torch.rand((708, 256, 3)), torch.rand((708, 4))  # data

class training_set(data.Dataset):
    def __init__(self,X,Y):
        self.X = X                           # set data
        self.Y = Y                           # set lables

    def __len__(self):
        return len(self.X)                   # return length

    def __getitem__(self, idx):
        return [self.X[idx], self.Y[idx]]    # return list of batch data [data, labels]

training_dataset = training_set(x_train, y_train)
train_loader = torch.utils.data.DataLoader(training_dataset, batch_size=50, shuffle=True)

Actually you don't need to use a custom data set because in your case it is simple dataset. You can first change to TensorDataset so that you can do like
training_dataset = torch.utils.data.TensorDataset(x_train, y_train)
train_loader = torch.utils.data.DataLoader(training_dataset, batch_size=50, shuffle=True)

both will return same results.


Instruction: how to vectorize the scatter-matmul operation

Input:
I have many matrices w1, w2, w3...wn with shapes (k*n1, k*n2, k*n3...k*nn) and x1, x2, x3...xn with shapes (n1*m, n2*m, n3*m...nn*m).
I want to get w1@x1, w2@x2, w3@x3 ... respectively.

The resulting matrix is multiple k*m matrices and can be concatenated into a large matrix with shape (k*n)*m.

Multiply them one by one will be slow. How to vectorize this operation?

Note: The input can be a k*(n1+n2+n3+...+nn) matrix and a (n1+n2+n3+...+nn)*m matrix, and we may use a batch index to indicate those submatrices.

This operation is related to the scatter operations implemented in pytorch_scatter, so I refer it as "scatter_matmul".

Output:
You can vectorize your operation by creating a large block-diagonal matrix W of shape n*kx(n1+..+nn) where the w_i matrices are the blocks on the diagonal. Then you can vertically stack all x matrices into an X matrix of shape (n1+..+nn)xm. Multiplying the block diagonal W with the vertical stack of all x matrices, X:

Y = W @ X

results with Y of shape (k*n)xm which is exactly the concatenated large matrix you are seeking.

If the shape of the block diagonal matrix W is too large to fit into memory, you may consider making W sparse and compute the product using torch.sparse.mm.


Instruction: What to define as entrypoint when initializing a pytorch estimator with a custom docker image for training on AWS Sagemaker?

Input:
So I created a docker image for training. In the dockerfile I have an entrypoint defined such that when docker run is executed, it will start running my python code.
To use this on aws sagemaker in my understanding I need to create a pytorch estimator in a jupyter notebook in sagemaker. I tried something like this:

import sagemaker
from sagemaker.pytorch import PyTorch

sagemaker_session = sagemaker.Session()

role = sagemaker.get_execution_role()

estimator = PyTorch(entry_point='train.py',
                    role=role,
                    framework_version='1.3.1',
                    image_name='xxx.ecr.eu-west-1.amazonaws.com/xxx:latest',
                    train_instance_count=1,
                    train_instance_type='ml.p3.xlarge',
                    hyperparameters={})

estimator.fit({})

In the documentation I found that as image name I can specify the link the my docker image on aws ecr. When I try to execute this it keeps complaining

[Errno 2] No such file or directory: 'train.py'

It complains immidiatly, so surely I am doing something completely wrong. I would expect that first my docker image should run, and than it could find out that the entry point does not exist.

But besides this, why do I need to specify an entry point, as in, should it not be clear that the entry to my training is simply docker run?

For maybe better understanding. The entrypoint python file in my docker image looks like this:

if __name__=='__main__':
    parser = argparse.ArgumentParser()

    # Hyperparameters sent by the client are passed as command-line arguments to the script.
    parser.add_argument('--epochs', type=int, default=5)
    parser.add_argument('--batch_size', type=int, default=16)
    parser.add_argument('--learning_rate', type=float, default=0.0001)

    # Data and output directories
    parser.add_argument('--output_data_dir', type=str, default=os.environ['OUTPUT_DATA_DIR'])
    parser.add_argument('--train_data_path', type=str, default=os.environ['CHANNEL_TRAIN'])
    parser.add_argument('--valid_data_path', type=str, default=os.environ['CHANNEL_VALID'])

    # Start training
    ...

Later I would like to specify the hyperparameters and data channels. But for now I simply do not understand what to put as entry point. In the documentation it says that the entrypoint is required and it should be a local/global path to the entrypoint...

Output:
If you really would like to use a complete separate by yourself build docker image, you should create an Amazon Sagemaker algorithm (which is one of the options in the Sagemaker menu). Here you have to specify a link to your docker image on amazon ECR as well as the input parameters and data channels etc. When choosing this options, you should not use the PyTorch estimater but the Algoritm estimater. This way you indeed don't have to specify an entrypoint because it simple runs the docker when training and the default entrypoint can be defined in your docker file.

The Pytorch estimator can be used when having you own model code, but you would like to run this code in an off-the-shelf Sagemaker PyTorch docker image. This is why you have to for example specify the PyTorch framework version. In this case the entrypoint file by default should be placed next to where your jupyter notebook is stored (just upload the file by clicking on the upload button). The PyTorch estimator inherits all options from the framework estimator where options can be found where to place the entrypoint and model, for example source_dir.


Instruction: Pytorch tensor, how to switch channel position - Runtime error

Input:
I have my training dataset as below, where X_train is 3D with 3 channels

Shape of X_Train:  (708, 256, 3)
Shape of Y_Train:  (708, 4)

Then I convert them into a tensor and input into the dataloader:

X_train=torch.from_numpy(X_data)
y_train=torch.from_numpy(y_data)
training_dataset = torch.utils.data.TensorDataset(X_train, y_train)
train_loader = torch.utils.data.DataLoader(training_dataset, batch_size=50, shuffle=False)

However when training the model, I get the following error:
RuntimeError: Given groups=1, weight of size 24 3 5, expected input[708, 256, 3] to have 3 channels, but got 256 channels instead

I suppose this is due to the position of the channel? In Tensorflow, the channel position is at the end, but in PyTorch the format is "Batch Size x Channel x Height x Width"? So how do I swap the positions in the x_train tensor to match the expected format in the dataloader?

class TwoLayerNet(torch.nn.Module):
    def __init__(self):
        super(TwoLayerNet,self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv1d(3, 3*8, kernel_size=5, stride=1),
            nn.Sigmoid(),
            nn.AvgPool1d(kernel_size=2, stride=0))
        self.conv2 = nn.Sequential(
            nn.Conv1d(3*8, 12, kernel_size=5, stride=1),
            nn.Sigmoid(),
            nn.AvgPool1d(kernel_size=2, stride = 0))
        #self.drop_out = nn.Dropout()

        self.fc1 = nn.Linear(708, 732)
        self.fc2 = nn.Linear(732, 4)

    def forward(self, x):
        out = self.conv1(x)
        out = self.conv2(out)
        out = out.reshape(out.size(0), -1)
        out = self.drop_out(out)
        out = self.fc1(out)
        out = self.fc2(out)
        return out

Output:
Use permute.

X_train = torch.rand(708, 256, 3)
X_train = X_train.permute(2, 0, 1)
X_train.shape
# => torch.Size([3, 708, 256])


Instruction: Unable to import TorchVision after installation Mac OSX

Input:
I've installed Pytorch and Torchvision in the way suggested on their website via pip within a virtual environment (env), and whilst no errors occur during installation when I go to import torchvision in my python code the following error occurs.

Traceback (most recent call last):
  File "demo.py", line 2, in <module>
    import torchvision
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/__init__.py", line 2, in <module>
    from torchvision import datasets
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/datasets/__init__.py", line 9, in <module>
    from .fakedata import FakeData
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/datasets/fakedata.py", line 3, in <module>
    from .. import transforms
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/transforms/__init__.py", line 1, in <module>
    from .transforms import *
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/transforms/transforms.py", line 17, in <module>
    from . import functional as F
  File "/Users/QuinceyBee/env/lib/python3.7/site-packages/torchvision/transforms/functional.py", line 5, in <module>
    from PIL import Image, ImageOps, ImageEnhance, PILLOW_VERSION
ImportError: cannot import name 'PILLOW_VERSION' from 'PIL' (/Users/QuinceyBee/env/lib/python3.7/site-packages/PIL/__init__.py)

I have tried creating new virtual environments to rebuild from scratch, tried to install via conda within a conda environment, however, neither of these resolved this issue.

I apologise for any format issues, this is the first time posting on here and also I'm relatively new to using python.

Any assistance would be greatly appreciated.

Output:
Pillow 7.0.0 removed PILLOW_VERSION, you should use version in your own code instead.

https://pillow.readthedocs.io/en/stable/deprecations.html#pillow-version-constant

If using Torchvision, there is a release planned this week (week 2, 2020) to fix it:

https://github.com/pytorch/vision/issues/1712#issuecomment-570286349

The options are:

wait for the new torchvision release
use the master version of torchvision as given below

pip install -U git+https://github.com/pytorch/vision)

install torchvision from a nightly, which also requires a pytorch from a nightly version
or install Pillow<7

pip install "pillow<7"


Instruction: Should this boxes for my custom Object Detector be smaller?

Input:
So Im trying to make a Object Detector for this companys forms, and we have labelled the images as shown in the example image I uploaded, my question is: Should We make more accurate boxes or is OK as they are, since the written part that we are trying to detect could be bigger.

So, what im asking is: In the example image, the "Descripcion" part or Description, has just 2 lines of text, but it could be more, should we make the box to just select the Description title + the 2 lines or so we stick to what we are doing now title + the 2 lines + all blank space that could have been filled with lines

Output:
It depends on what you really want to do with the detected boxes. What are the next steps, can the next step e.g. extracting the text handle all the free space, or would it be better to just the the part where it is actually written.

Besides that right now in your example I find that most boxes are too big. The form is more or less already splitted in boxes and it could be better to make the boxes smaller and more accurate e.g.the box around IMPORTE and some amount in €. I would label this closer. So the box only contains the information you actually want and nothing else.

But as I said it really depends on the next step the boxes should be used for.


Instruction: coremltools: how to properly use NeuralNetworkMultiArrayShapeRange?

Input:
I have a PyTorch network and I want to deploy it to iOS devices. In short, I fail to add flexibility to the input tensor shape in CoreML.

The network is a convnet that takes an RGB image (stored as a tensor) as an input and returns an RGB image of the same size. Using PyTorch, I can input images of any size I want, for instance a tensor of size (1, 3, 300, 300) for a 300x300 image.

To convert the PyTorch model to a CoreML model, I first convert it to an ONNX model using torch.onnx.export. This function requires to pass a dummy input so that it can execute the graph. So I did using:

input = torch.rand(1, 3, 300, 300)

My guess is that the ONNX model only accepts images / tensors of size (1, 3, 300, 300). Now, I can use the onnx_coreml.convert function to convert the ONNX model to a CoreML model. By printing the CoreML model's spec description using Python, I get something like:

input {
  name: "my_image"
  type {
    multiArrayType {
      shape: 1
      shape: 3
      shape: 300
      shape: 300
      dataType: FLOAT32
    }
  }
}
output {
  name: "my_output"
  type {
    multiArrayType {
      shape: 1
      shape: 3
      shape: 300
      shape: 300
      dataType: FLOAT32
    }
  }
}
metadata {
  userDefined {
    key: "coremltoolsVersion"
    value: "3.1"
  }
}

The model's input must be a multiArrayType of size (1, 3, 300, 300). By copying this model to XCode, I can see while inspecting the model that my_name is listed under the "Inputs" section and it is expected to be a MultiArray (Float32 1 x 3 x 300 x 300). So far, everything is coherent.

My problem is to add flexibility to the input shape. I tried to use coremltools with no luck. This is my problem. Here is my code:

import coremltools
from coremltools.models.neural_network import flexible_shape_utils

spec = coremltools.utils.load_spec('my_model.mlmodel')

shape_range = flexible_shape_utils.NeuralNetworkMultiArrayShapeRange()
shape_range.add_channel_range((3,3))
shape_range.add_height_range((64, 5000))
shape_range.add_width_range((64, 5000))
flexible_shape_utils.update_multiarray_shape_range(spec, feature_name='my_image', shape_range=shape_range)
coremltools.models.utils.save_spec(spec, 'my_flexible_model.mlmodel')

I get the following spec description using Python:

input {
  name: "my_image"
  type {
    multiArrayType {
      shape: 1
      shape: 3
      shape: 300
      shape: 300
      dataType: FLOAT32
      shapeRange {
        sizeRanges {
          lowerBound: 3
          upperBound: 3
        }
        sizeRanges {
          lowerBound: 64
          upperBound: 5000
        }
        sizeRanges {
          lowerBound: 64
          upperBound: 5000
        }
      }
    }
  }
}

Only 3 ranges as specified, which makes sense since I only defined a range for channel, height and width, but not for the batch size. In XCode, I get the following error when inspecting the flexible CoreML model:

There was a problem decoding this CoreML document
validator error: Description of multiarray feature 'my_image' has a default 4-d shape but a 3-d shape range

I'm pretty sure it was working on another project when I was on macOS X Mojave, but at this point I'm sure of nothing.

I'm using:

macOS X Catalina
conda 4.7.12
python 3.7.5
pytorch 1.3.1
onnx 1.6.0
onnx-coreml 1.1
coremltools 3.1

Thanks for the help

Output:
Easiest thing to do is to remove that shape:1. Something like this:

del spec.description.input[0].shape[0]

Now the default shape should also have 3 dimensions.

However, I would suggest changing the type of the input from multi-array to an actual image. Since you're going to be using it with images anyway. That will let you pass in the image as a CVPixelBuffer or CGImage object instead of an MLMultiArray.


Instruction: In Torch C++ API, How to write to the internal data of a tensor fastly?

Input:
I am using torch C++ frontend and want to have a tensor with specified value in it. To achieve this one may allocate memory and set value by hand, then use torch::from_blob to build a tensor on the memory block, but it seems not clean enough for me.

In the very bottom of this document I found out that I can use subscript to directly access and modify the data. However, this approach has a big running time overhead, likely because the subscript access will treat the element of tensor as a 0-d tensor. The following code will cost more than 2 seconds on my machine (-O3 optimization level), which is unreasonably long for modern CPU.

    torch::Tensor tensor = torch::empty({1000, 1000});
    for(int i=0; i < 1000; i++)
    {
        for(int j=0 ; j < 1000; j++)
        {
            tensor[i][j] = calc_tensor_data(i,j);
        }
    }

Is there a clean and fast way to achieve this goal?

Output:
After hours of fruitless search on the Internet, I got a hypothesis in my mind, and I decided to give it a shot. It turns out that the accessor mentioned in the same document works well as left value too, although this feature is not mentioned by document at all. The following code is just fine, and it is as fast as manipulating raw pointer directly.

    torch::Tensor tensor = torch::empty({1000, 1000});
    auto accessor = tensor.accessor<float,2>();
    for(int i=0; i < 1000; i++)
    {
        for(int j=0 ; j < 1000; j++)
        {
            accessor[i][j] = calc_tensor_data(i,j);
        }
    }


Instruction: Pytorch : AttributeError: 'function' object has no attribute 'cuda'

Input:
import torch
import models
model_names = sorted(name for name in models.__dict__
                     if name.islower() and not name.startswith("__")
                     and callable(models.__dict__[name]))
model = models.__dict__['resnet18']
model = torch.nn.DataParallel(model,device_ids = [0])  #PROBLEM CAUSING LINE
model.to('cuda:0')

To run this code you need to clone this repository : https://github.com/SoftwareGift/FeatherNets_Face-Anti-spoofing-Attack-Detection-Challenge-CVPR2019.git

Please run this piece of code inside the root folder of the cloned directory.

I am getting the follow error AttributeError: 'function' object has no attribute 'cuda'
I have tried using torch.device object as well for the same function and it results in the same error.
Please ask for any other details that are required. PyTorch newbie here
python:3.7 pytorch:1.3.1

Output:
Replace

model = torch.nn.DataParallel(model,device_ids = [0])

with

model = torch.nn.DataParallel(model(), device_ids=[0])

(notice the () after model inside DataParallel). The difference is simple: the models module contains classes/functions which create models and not instances of models. If you trace the imports, you'll find that models.__dict__['resnet18'] resolves to this function. Since DataParallel wraps an instance, not a class itself, it is incompatible. The () calls this model building function/class constructor to create an instance of this model.

A much simpler example of this would be the following

class MyNet(nn.Model):
    def __init__(self):
        self.linear = nn.Linear(4, 4)
    def forward(self, x):
        return self.linear(x)

model = nn.DataParallel(MyNet) # this is what you're doing
model = nn.DataParallel(MyNet()) # this is what you should be doing

Your error message complains that function (since model without the () is of type function) has no attribute cuda, which is a method of nn.Model instances.


Instruction: How to Debug Saving Model TypeError: can't pickle SwigPyObject objects?

Input:
I'm trying to save a model, which is an object of a class that inherits from nn.Module. It trains fine without problem, but when I try running the code:

    torch.save(
        obj=model,
        f=os.path.join(tensorboard_writer.get_logdir(), 'model.ckpt'))

I receive the error: TypeError: can't pickle SwigPyObject objects

I have no idea what a SwigPyObject object is. How do I debug this error to save my model?

Full Traceback:

Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 149, in _with_file_like
    return body(f)
  File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 224, in <lambda>
    return _with_file_like(f, "wb", lambda f: _save(obj, f, pickle_module, pickle_protocol))
  File "/usr/local/lib/python3.6/dist-packages/torch/serialization.py", line 296, in _save
    pickler.dump(obj)
TypeError: can't pickle SwigPyObject objects

In case it helps, my model is an object of the following class:

class RecurrentModel(nn.Module):

    def __init__(self,
                 core_str,
                 core_kwargs,
                 tensorboard_writer=None,
                 input_size=1,
                 hidden_size=32,
                 output_size=2):

        super(RecurrentModel, self).__init__()
        self.tensorboard_writer = tensorboard_writer
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size

        self.core = self._create_core(core_str=core_str, core_kwargs=core_kwargs)
        self.core_hidden = None
        self.linear = nn.Linear(
            in_features=hidden_size,
            out_features=output_size,
            bias=True)
        self.softmax = nn.Softmax(dim=-1)

        # converts all weights into doubles i.e. float64
        # this prevents PyTorch from breaking when multiplying float32 * flaot64
        self.double()

        # TODO figure out why writing the model to tensorboard doesn't work
        # dummy_input = torch.zeros(size=(10, 1, 1), dtype=torch.double)
        # tensorboard_writer.add_graph(
        #     model=self,
        #     input_to_model=dict(stimulus=dummy_input))

    def _create_core(self, core_str, core_kwargs):
        if core_str == 'lstm':
            core_constructor = nn.LSTM
        elif core_str == 'rnn':
            core_constructor = nn.RNN
        elif core_str == 'gru':
            core_constructor = nn.GRU
        else:
            raise ValueError('Unknown core string')

        core = core_constructor(
            input_size=self.input_size,
            hidden_size=self.hidden_size,
            batch_first=True,
            **core_kwargs)
        return core

    def forward(self, model_input):

        if self.core_hidden is None:
            core_output, self.core_hidden = self.core(
                model_input['stimulus'])
        else:
            core_output, self.core_hidden = self.core(
                model_input['stimulus'],
                self.core_hidden)

        linear_output = self.linear(core_output)

        softmax_output = self.softmax(linear_output)

        forward_output = dict(
            core_output=core_output,
            core_hidden=self.core_hidden,
            linear_output=linear_output,
            softmax_output=softmax_output)

        return forward_output

    def reset_core_hidden(self):
        self.core_hidden = None

I then construct and save the model:

    tensorboard_writer = SummaryWriter()

    model = RecurrentModel(
        core_str='rnn',
        core_kwargs={},
        tensorboard_writer=tensorboard_writer)

    torch.save(
        obj=model,
        f=os.path.join(tensorboard_writer.get_logdir(), 'model.ckpt')
    )

Output:
Someone on the PyTorch forum clarified. SummaryWriter has a file handle, which is typically unable to serialize. The problematic line is:

self.tensorboard_writer = tensorboard_writer

https://github.com/pytorch/pytorch/issues/32046#issuecomment-573151743


Instruction: Testing and Confidence score of Network trained with nn.CrossEntropyLoss()

Input:
I have trained a network with the following structure:

Intent_LSTM(
(attention): Attention()
(embedding): Embedding(34601, 400)
(lstm): LSTM(400, 512, num_layers=2, batch_first=True, dropout=0.5)
(dropout): Dropout(p=0.5, inplace=False)
(fc): Linear(in_features=512, out_features=3, bias=True)
)

Now I want to test this trained network and also get the confidence score of classification.
Here is my current implementation of test function:

output = model_current(inputs)
pred = torch.round(output.squeeze())
pred = pred.argmax(dim=1, keepdim=True)

Now my question isas follows.

Here pred is just the output from a Fully connected layer from my network without softmax (as required by a loss function). Is this(pred = pred.argmax(dim=1, keepdim=True)) the right way to get the predictions? Or should I pass the output from the network to a softmax layer and then do argmax?
How do I get the confidence score? Should I pass the output from the network to a softmax layer and select the argmax as the confidence of the class?

Output:
It doesn't really matter if you pick argmax before or after doing softmax. Because whatever maximizes softmax will also maximize the logits (pre-softmax) values. So you should get similar values.
Softmax will give you scores or probabilities for each class. Hence, the values after doing softmax can be used as confidence scores.


Instruction: autograd differentiation example in PyTorch - should be 9/8?

Input:
In the example for the Torch tutorial for Python, they use the following graph:

x = [[1, 1], [1, 1]]
y = x + 2
z = 3y^2
o = mean( z )  # 1/4 * x.sum()

Thus, the forward pass gets us this:

x_i = 1, y_i = 3, z_i = 27, o = 27

In code this looks like:

import torch

# define graph
x = torch.ones(2, 2, requires_grad=True)
y = x + 2
z = y * y * 3
out = z.mean()

# if we don't do this, torch will only retain gradients for leaf nodes, ie: x
y.retain_grad()
z.retain_grad()

# does a forward pass
print(z, out)

however, I get confused at the gradients computed:

# now let's run our backward prop & get gradients
out.backward()
print(f'do/dz = {z.grad[0,0]}')

which outputs:

do/dx = 4.5

By chain rule, do/dx = do/dz * dz/dy * dy/dx, where:

dy/dx = 1
dz/dy = 9/2 given x_i=1
do/dz = 1/4 given x_i=1

which means:

do/dx = 1/4 * 9/2 * 1 = 9/8

However this doesn't match the gradients returned by Torch (9/2 = 4.5). Perhaps I have a math error (something with the do/dz = 1/4 term?), or I don't understand autograd in Torch.

Any pointers?

Output:
do/dz = 1 / 4
dz/dy = 6y = 6 * 3 = 18
dy/dx = 1

therefore, do/dx = 9/2