Instruction: How can I compute number of FLOPs and Params for 1-d CNN? Use pytorch platform

Input:
My network is a 1d CNN, I want to compute the number of FLOPs and params. I used public method 'flops_counter', but I am not sure the size of the input. When I run it with size(128,1,50), I get error 'Expected 3-dimensional input for 3-dimensional weight [128, 1, 50], but got 4-dimensional input of size [1, 128, 1, 50] instead'. When I run it with size(128,50), I get error 'RuntimeError: Given groups=1, weight of size [128, 1, 50], expected input[1, 128, 50] to have 1 channels, but got 128 channels instead'.
import torch
from models.cnn import net
from flops_counter import get_model_complexity_info

model = net()
# Flops&params
flops, params = get_model_complexity_info(model, (128,1,50), as_strings=True, print_per_layer_stat=True)
print('Flops:  ' + flops)
print('Params: ' + params)

Here is my 1d CNN.
from __future__ import print_function
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

# create cnn
class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.conv1 = nn.Conv1d(1, 128, 50, stride=3)
        self.conv2 = nn.Conv1d(128, 32, 7, stride=1)
        self.conv3 = nn.Conv1d(32, 32, 9, stride=1)
        self.fc1 = nn.Linear(32, 128)
        self.fc2 = nn.Linear(128, 5)

        self.bn1 = nn.BatchNorm1d(128)
        self.bn2 = nn.BatchNorm1d(32)

        self.dropout = nn.Dropout2d(0.5)
        self.faltten = nn.Flatten()

    # forward propagation
    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = self.bn1(x)
        x = F.max_pool1d(x, 2, stride=3)

        x = self.dropout(F.relu(self.conv2(x)))
        # x = F.relu(self.conv2(x))
        x = self.bn2(x)
        x = F.max_pool1d(x, 2, stride=2)

        x = self.dropout(F.relu(self.conv3(x)))

        x = self.faltten(x)
        x = self.dropout(self.fc1(x))
        output = self.fc2(x)

        return output

Here is the code of flops_counter:
'''
Copyright (C) 2019 Sovrasov V. - All Rights Reserved
 * You may use, distribute and modify this code under the
 * terms of the MIT license.
 * You should have received a copy of the MIT license with
 * this file. If not visit https://opensource.org/licenses/MIT
'''

import sys
from functools import partial

import torch
import torch.nn as nn
import numpy as np

def get_model_complexity_info(model, input_res,
                              print_per_layer_stat=True,
                              as_strings=True,
                              input_constructor=None, ost=sys.stdout,
                              verbose=False, ignore_modules=[],
                              custom_modules_hooks={}):
    assert type(input_res) is tuple
    assert len(input_res) >= 1
    assert isinstance(model, nn.Module)
    global CUSTOM_MODULES_MAPPING
    CUSTOM_MODULES_MAPPING = custom_modules_hooks
    flops_model = add_flops_counting_methods(model)
    flops_model.eval()
    flops_model.start_flops_count(ost=ost, verbose=verbose, ignore_list=ignore_modules)
    if input_constructor:
        input = input_constructor(input_res)
        _ = flops_model(**input)
    else:
        try:
            batch = torch.ones(()).new_empty((1, *input_res),
                                             dtype=next(flops_model.parameters()).dtype,
                                             device=next(flops_model.parameters()).device)
        except StopIteration:
            batch = torch.ones(()).new_empty((1, *input_res))

        _ = flops_model(batch)

    flops_count, params_count = flops_model.compute_average_flops_cost()
    if print_per_layer_stat:
        print_model_with_flops(flops_model, flops_count, params_count, ost=ost)
    flops_model.stop_flops_count()
    CUSTOM_MODULES_MAPPING = {}

    if as_strings:
        return flops_to_string(flops_count), params_to_string(params_count)

    return flops_count, params_count

def flops_to_string(flops, units='GMac', precision=2):
    if units is None:
        if flops // 10**9 > 0:
            return str(round(flops / 10.**9, precision)) + ' GMac'
        elif flops // 10**6 > 0:
            return str(round(flops / 10.**6, precision)) + ' MMac'
        elif flops // 10**3 > 0:
            return str(round(flops / 10.**3, precision)) + ' KMac'
        else:
            return str(flops) + ' Mac'
    else:
        if units == 'GMac':
            return str(round(flops / 10.**9, precision)) + ' ' + units
        elif units == 'MMac':
            return str(round(flops / 10.**6, precision)) + ' ' + units
        elif units == 'KMac':
            return str(round(flops / 10.**3, precision)) + ' ' + units
        else:
            return str(flops) + ' Mac'

def params_to_string(params_num, units=None, precision=2):
    if units is None:
        if params_num // 10 ** 6 > 0:
            return str(round(params_num / 10 ** 6, 2)) + ' M'
        elif params_num // 10 ** 3:
            return str(round(params_num / 10 ** 3, 2)) + ' k'
        else:
            return str(params_num)
    else:
        if units == 'M':
            return str(round(params_num / 10.**6, precision)) + ' ' + units
        elif units == 'K':
            return str(round(params_num / 10.**3, precision)) + ' ' + units
        else:
            return str(params_num)

def print_model_with_flops(model, total_flops, total_params, units='GMac',
                           precision=3, ost=sys.stdout):

    def accumulate_params(self):
        if is_supported_instance(self):
            return self.__params__
        else:
            sum = 0
            for m in self.children():
                sum += m.accumulate_params()
            return sum

    def accumulate_flops(self):
        if is_supported_instance(self):
            return self.__flops__ / model.__batch_counter__
        else:
            sum = 0
            for m in self.children():
                sum += m.accumulate_flops()
            return sum

    def flops_repr(self):
        accumulated_params_num = self.accumulate_params()
        accumulated_flops_cost = self.accumulate_flops()
        return ', '.join([params_to_string(accumulated_params_num, units='M', precision=precision),
                          '{:.3%} Params'.format(accumulated_params_num / total_params),
                          flops_to_string(accumulated_flops_cost, units=units, precision=precision),
                          '{:.3%} MACs'.format(accumulated_flops_cost / total_flops),
                          self.original_extra_repr()])

    def add_extra_repr(m):
        m.accumulate_flops = accumulate_flops.__get__(m)
        m.accumulate_params = accumulate_params.__get__(m)
        flops_extra_repr = flops_repr.__get__(m)
        if m.extra_repr != flops_extra_repr:
            m.original_extra_repr = m.extra_repr
            m.extra_repr = flops_extra_repr
            assert m.extra_repr != m.original_extra_repr

    def del_extra_repr(m):
        if hasattr(m, 'original_extra_repr'):
            m.extra_repr = m.original_extra_repr
            del m.original_extra_repr
        if hasattr(m, 'accumulate_flops'):
            del m.accumulate_flops

    model.apply(add_extra_repr)
    print(model, file=ost)
    model.apply(del_extra_repr)

def get_model_parameters_number(model):
    params_num = sum(p.numel() for p in model.parameters() if p.requires_grad)
    return params_num

def add_flops_counting_methods(net_main_module):
    # adding additional methods to the existing module object,
    # this is done this way so that each function has access to self object
    net_main_module.start_flops_count = start_flops_count.__get__(net_main_module)
    net_main_module.stop_flops_count = stop_flops_count.__get__(net_main_module)
    net_main_module.reset_flops_count = reset_flops_count.__get__(net_main_module)
    net_main_module.compute_average_flops_cost = compute_average_flops_cost.__get__(net_main_module)

    net_main_module.reset_flops_count()

    return net_main_module

def compute_average_flops_cost(self):
    """
    A method that will be available after add_flops_counting_methods() is called
    on a desired net object.
    Returns current mean flops consumption per image.
    """

    batches_count = self.__batch_counter__
    flops_sum = 0
    params_sum = 0
    for module in self.modules():
        if is_supported_instance(module):
            flops_sum += module.__flops__
    params_sum = get_model_parameters_number(self)
    return flops_sum / batches_count, params_sum

def start_flops_count(self, **kwargs):
    """
    A method that will be available after add_flops_counting_methods() is called
    on a desired net object.
    Activates the computation of mean flops consumption per image.
    Call it before you run the network.
    """
    add_batch_counter_hook_function(self)

    seen_types = set()
    def add_flops_counter_hook_function(module, ost, verbose, ignore_list):
        if type(module) in ignore_list:
            seen_types.add(type(module))
            if is_supported_instance(module):
                module.__params__ = 0
        elif is_supported_instance(module):
            if hasattr(module, '__flops_handle__'):
                return
            if type(module) in CUSTOM_MODULES_MAPPING:
                handle = module.register_forward_hook(CUSTOM_MODULES_MAPPING[type(module)])
            else:
                handle = module.register_forward_hook(MODULES_MAPPING[type(module)])
            module.__flops_handle__ = handle
            seen_types.add(type(module))
        else:
            if verbose and not type(module) in (nn.Sequential, nn.ModuleList) and not type(module) in seen_types:
                print('Warning: module ' + type(module).__name__ + ' is treated as a zero-op.', file=ost)
            seen_types.add(type(module))

    self.apply(partial(add_flops_counter_hook_function, **kwargs))

def stop_flops_count(self):
    """
    A method that will be available after add_flops_counting_methods() is called
    on a desired net object.
    Stops computing the mean flops consumption per image.
    Call whenever you want to pause the computation.
    """
    remove_batch_counter_hook_function(self)
    self.apply(remove_flops_counter_hook_function)

def reset_flops_count(self):
    """
    A method that will be available after add_flops_counting_methods() is called
    on a desired net object.
    Resets statistics computed so far.
    """
    add_batch_counter_variables_or_reset(self)
    self.apply(add_flops_counter_variable_or_reset)

# ---- Internal functions
def empty_flops_counter_hook(module, input, output):
    module.__flops__ += 0

def upsample_flops_counter_hook(module, input, output):
    output_size = output[0]
    batch_size = output_size.shape[0]
    output_elements_count = batch_size
    for val in output_size.shape[1:]:
        output_elements_count *= val
    module.__flops__ += int(output_elements_count)

def relu_flops_counter_hook(module, input, output):
    active_elements_count = output.numel()
    module.__flops__ += int(active_elements_count)

def linear_flops_counter_hook(module, input, output):
    input = input[0]
    output_last_dim = output.shape[-1]  # pytorch checks dimensions, so here we don't care much
    module.__flops__ += int(np.prod(input.shape) * output_last_dim)

def pool_flops_counter_hook(module, input, output):
    input = input[0]
    module.__flops__ += int(np.prod(input.shape))

def bn_flops_counter_hook(module, input, output):
    module.affine
    input = input[0]

    batch_flops = np.prod(input.shape)
    if module.affine:
        batch_flops *= 2
    module.__flops__ += int(batch_flops)

def deconv_flops_counter_hook(conv_module, input, output):
    # Can have multiple inputs, getting the first one
    input = input[0]

    batch_size = input.shape[0]
    input_height, input_width = input.shape[2:]

    kernel_height, kernel_width = conv_module.kernel_size
    in_channels = conv_module.in_channels
    out_channels = conv_module.out_channels
    groups = conv_module.groups

    filters_per_channel = out_channels // groups
    conv_per_position_flops = kernel_height * kernel_width * in_channels * filters_per_channel

    active_elements_count = batch_size * input_height * input_width
    overall_conv_flops = conv_per_position_flops * active_elements_count
    bias_flops = 0
    if conv_module.bias is not None:
        output_height, output_width = output.shape[2:]
        bias_flops = out_channels * batch_size * output_height * output_height
    overall_flops = overall_conv_flops + bias_flops

    conv_module.__flops__ += int(overall_flops)

def conv_flops_counter_hook(conv_module, input, output):
    # Can have multiple inputs, getting the first one
    input = input[0]

    batch_size = input.shape[0]
    output_dims = list(output.shape[2:])

    kernel_dims = list(conv_module.kernel_size)
    in_channels = conv_module.in_channels
    out_channels = conv_module.out_channels
    groups = conv_module.groups

    filters_per_channel = out_channels // groups
    conv_per_position_flops = int(np.prod(kernel_dims)) * in_channels * filters_per_channel

    active_elements_count = batch_size * int(np.prod(output_dims))

    overall_conv_flops = conv_per_position_flops * active_elements_count

    bias_flops = 0

    if conv_module.bias is not None:

        bias_flops = out_channels * active_elements_count

    overall_flops = overall_conv_flops + bias_flops

    conv_module.__flops__ += int(overall_flops)

def batch_counter_hook(module, input, output):
    batch_size = 1
    if len(input) > 0:
        # Can have multiple inputs, getting the first one
        input = input[0]
        batch_size = len(input)
    else:
        pass
        print('Warning! No positional inputs found for a module, assuming batch size is 1.')
    module.__batch_counter__ += batch_size

def rnn_flops(flops, rnn_module, w_ih, w_hh, input_size):
    # matrix matrix mult ih state and internal state
    flops += w_ih.shape[0]*w_ih.shape[1]
    # matrix matrix mult hh state and internal state
    flops += w_hh.shape[0]*w_hh.shape[1]
    if isinstance(rnn_module, (nn.RNN, nn.RNNCell)):
        # add both operations
        flops += rnn_module.hidden_size
    elif isinstance(rnn_module, (nn.GRU, nn.GRUCell)):
        # hadamard of r
        flops += rnn_module.hidden_size
        # adding operations from both states
        flops += rnn_module.hidden_size*3
        # last two hadamard product and add
        flops += rnn_module.hidden_size*3
    elif isinstance(rnn_module, (nn.LSTM, nn.LSTMCell)):
        # adding operations from both states
        flops += rnn_module.hidden_size*4
        # two hadamard product and add for C state
        flops += rnn_module.hidden_size + rnn_module.hidden_size + rnn_module.hidden_size
        # final hadamard
        flops += rnn_module.hidden_size + rnn_module.hidden_size + rnn_module.hidden_size
    return flops

def rnn_flops_counter_hook(rnn_module, input, output):
    """
    Takes into account batch goes at first position, contrary
    to pytorch common rule (but actually it doesn't matter).
    IF sigmoid and tanh are made hard, only a comparison FLOPS should be accurate
    """
    flops = 0
    inp = input[0] # input is a tuble containing a sequence to process and (optionally) hidden state
    batch_size = inp.shape[0]
    seq_length = inp.shape[1]
    num_layers = rnn_module.num_layers

    for i in range(num_layers):
        w_ih = rnn_module.__getattr__('weight_ih_l' + str(i))
        w_hh = rnn_module.__getattr__('weight_hh_l' + str(i))
        if i == 0:
            input_size = rnn_module.input_size
        else:
            input_size = rnn_module.hidden_size
        flops = rnn_flops(flops, rnn_module, w_ih, w_hh, input_size)
        if rnn_module.bias:
            b_ih = rnn_module.__getattr__('bias_ih_l' + str(i))
            b_hh = rnn_module.__getattr__('bias_hh_l' + str(i))
            flops += b_ih.shape[0] + b_hh.shape[0]

    flops *= batch_size
    flops *= seq_length
    if rnn_module.bidirectional:
        flops *= 2
    rnn_module.__flops__ += int(flops)

def rnn_cell_flops_counter_hook(rnn_cell_module, input, output):
    flops = 0
    inp = input[0]
    batch_size = inp.shape[0]
    w_ih = rnn_cell_module.__getattr__('weight_ih')
    w_hh = rnn_cell_module.__getattr__('weight_hh')
    input_size = inp.shape[1]
    flops = rnn_flops(flops, rnn_cell_module, w_ih, w_hh, input_size)
    if rnn_cell_module.bias:
        b_ih = rnn_cell_module.__getattr__('bias_ih')
        b_hh = rnn_cell_module.__getattr__('bias_hh')
        flops += b_ih.shape[0] + b_hh.shape[0]

    flops *= batch_size
    rnn_cell_module.__flops__ += int(flops)

def add_batch_counter_variables_or_reset(module):

    module.__batch_counter__ = 0

def add_batch_counter_hook_function(module):
    if hasattr(module, '__batch_counter_handle__'):
        return

    handle = module.register_forward_hook(batch_counter_hook)
    module.__batch_counter_handle__ = handle

def remove_batch_counter_hook_function(module):
    if hasattr(module, '__batch_counter_handle__'):
        module.__batch_counter_handle__.remove()
        del module.__batch_counter_handle__

def add_flops_counter_variable_or_reset(module):
    if is_supported_instance(module):
        if hasattr(module, '__flops__') or hasattr(module, '__params__'):
            print('Warning: variables __flops__ or __params__ are already '
                    'defined for the module' + type(module).__name__ +
                    ' ptflops can affect your code!')
        module.__flops__ = 0
        module.__params__ = get_model_parameters_number(module)

CUSTOM_MODULES_MAPPING = {}

MODULES_MAPPING = {
    # convolutions
    nn.Conv1d: conv_flops_counter_hook,
    nn.Conv2d: conv_flops_counter_hook,
    nn.Conv3d: conv_flops_counter_hook,
    # activations
    nn.ReLU: relu_flops_counter_hook,
    nn.PReLU: relu_flops_counter_hook,
    nn.ELU: relu_flops_counter_hook,
    nn.LeakyReLU: relu_flops_counter_hook,
    nn.ReLU6: relu_flops_counter_hook,
    # poolings
    nn.MaxPool1d: pool_flops_counter_hook,
    nn.AvgPool1d: pool_flops_counter_hook,
    nn.AvgPool2d: pool_flops_counter_hook,
    nn.MaxPool2d: pool_flops_counter_hook,
    nn.MaxPool3d: pool_flops_counter_hook,
    nn.AvgPool3d: pool_flops_counter_hook,
    nn.AdaptiveMaxPool1d: pool_flops_counter_hook,
    nn.AdaptiveAvgPool1d: pool_flops_counter_hook,
    nn.AdaptiveMaxPool2d: pool_flops_counter_hook,
    nn.AdaptiveAvgPool2d: pool_flops_counter_hook,
    nn.AdaptiveMaxPool3d: pool_flops_counter_hook,
    nn.AdaptiveAvgPool3d: pool_flops_counter_hook,
    # BNs
    nn.BatchNorm1d: bn_flops_counter_hook,
    nn.BatchNorm2d: bn_flops_counter_hook,
    nn.BatchNorm3d: bn_flops_counter_hook,
    # FC
    nn.Linear: linear_flops_counter_hook,
    # Upscale
    nn.Upsample: upsample_flops_counter_hook,
    # Deconvolution
    nn.ConvTranspose2d: deconv_flops_counter_hook,
    # RNN
    nn.RNN: rnn_flops_counter_hook,
    nn.GRU: rnn_flops_counter_hook,
    nn.LSTM: rnn_flops_counter_hook,
    nn.RNNCell: rnn_cell_flops_counter_hook,
    nn.LSTMCell: rnn_cell_flops_counter_hook,
    nn.GRUCell: rnn_cell_flops_counter_hook
}

def is_supported_instance(module):
    if type(module) in MODULES_MAPPING or type(module) in CUSTOM_MODULES_MAPPING:
        return True
    return False

def remove_flops_counter_hook_function(module):
    if is_supported_instance(module):
        if hasattr(module, '__flops_handle__'):
            module.__flops_handle__.remove()
            del module.__flops_handle__

Output:
Here is working code using the ptflops package. You need to take care of the length of your input sequence. The pytorch doc for Conv1d reads:  ,
which lets you backtrace the input size you need from the first fully connected layer (see my comments in the model definition).
from ptflops import get_model_complexity_info
import torch.nn as nn
import torch.nn.functional as F

class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.conv1 = nn.Conv1d(1, 128, 50, stride=3)  # Lin = 260
        # max_pool1d(x, 2, stride=3)  # Lin = 71
        self.conv2 = nn.Conv1d(128, 32, 7, stride=1)  # Lin = 24
        # max_pool1d(x, 2, stride=2)  # Lin = 18
        self.conv3 = nn.Conv1d(32, 32, 9, stride=1)  # Lin = 9
        self.fc1 = nn.Linear(32, 128)
        self.fc2 = nn.Linear(128, 5)

        self.bn1 = nn.BatchNorm1d(128)
        self.bn2 = nn.BatchNorm1d(32)

        self.dropout = nn.Dropout2d(0.5)
        self.flatten = nn.Flatten()

    # forward propagation
    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = self.bn1(x)
        x = F.max_pool1d(x, 2, stride=3)

        x = self.dropout(F.relu(self.conv2(x)))
        # x = F.relu(self.conv2(x))
        x = self.bn2(x)
        x = F.max_pool1d(x, 2, stride=2)

        x = self.dropout(F.relu(self.conv3(x)))

        x = self.flatten(x)
        x = self.dropout(self.fc1(x))
        output = self.fc2(x)

        return output

macs, params = get_model_complexity_info(net(), (1, 260), as_strings=False,
                                         print_per_layer_stat=True, verbose=True)
print('{:<30}  {:<8}'.format('Computational complexity: ', macs))
print('{:<30}  {:<8}'.format('Number of parameters: ', params))

output:
net(
  0.05 M, 100.000% Params, 0.001 GMac, 100.000% MACs,
  (conv1): Conv1d(0.007 M, 13.143% Params, 0.0 GMac, 45.733% MACs, 1, 128, kernel_size=(50,), stride=(3,))
  (conv2): Conv1d(0.029 M, 57.791% Params, 0.001 GMac, 50.980% MACs, 128, 32, kernel_size=(7,), stride=(1,))
  (conv3): Conv1d(0.009 M, 18.619% Params, 0.0 GMac, 0.913% MACs, 32, 32, kernel_size=(9,), stride=(1,))
  (fc1): Linear(0.004 M, 8.504% Params, 0.0 GMac, 0.404% MACs, in_features=32, out_features=128, bias=True)
  (fc2): Linear(0.001 M, 1.299% Params, 0.0 GMac, 0.063% MACs, in_features=128, out_features=5, bias=True)
  (bn1): BatchNorm1d(0.0 M, 0.515% Params, 0.0 GMac, 1.793% MACs, 128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (bn2): BatchNorm1d(0.0 M, 0.129% Params, 0.0 GMac, 0.114% MACs, 32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (dropout): Dropout2d(0.0 M, 0.000% Params, 0.0 GMac, 0.000% MACs, p=0.5, inplace=False)
  (flatten): Flatten(0.0 M, 0.000% Params, 0.0 GMac, 0.000% MACs, )
)
Computational complexity:       1013472.0
Number of parameters:           49669
```


Instruction: PyTorch: Perform add/sub/mul operations using 1-D tensor and multi-channel (3-D) image tensor

Input:
Note: I am looking for the fastest/optimal way of doing or improving on this, as my constraint is time
Using PyTorch, I have an image as a 3D tensor, lets say of dimension 64 x 400 x 400, where 64 refers to channels and 400 is the image dimensions. With this, I have a 64 length 1D tensor, all with different values, where the intention is to use one value per channel. I want to use a value per 1D tensor to apply to the entire 400x400 block of the channel. So for example, when I want to add 3d_tensor + 1d_tensor, I want 1d_tensor[i] to be added to all 400x400 = 160000 values in 3d_tensor[i], with [i] ranging from 0 to 63.
What I previously did:
I tried doing it directly, by using the operators only:
output_add = 1d_tensor + 3d_tensor

This returned an error that the dimension of 3d_tensor (400) and 1d_tensor (64) are incompatible.
So my current form is using a for loop
for a, b in zip(3d_tensor, 1d_tensor):
   a += b

However at one stage I have four different 1D tensors to use at once in either addition, subtraction or multiplication, so is this for loop method the most efficient? I'm also planning on doing it 20+ times per image, so speed is key. I tried maybe extending the 1d tensor to a dimension of 64 x 400 x 400 also, so it could be used directly, but I could not get this right using tensor.repeat()

Output:
You should add some dimension to the 1D array: convert from 64 to (64 x 1 x 1):
output_add = 1d_tensor[:, None, None] + 3d_tensor

with this None type indexing you can add dimension any where.
The [:, None, None] will add two additional dimension to the 1D array after the existing dimension.
Or you can use view for the same result see:
output_add = 1d_tensor.view(-1, 1, 1) + 3d_tensor

The result has the same dimension as the 3D array: (64, 1, 1). So the pytorch can use broadcasting.
Here a good explanation for broad casting: How does pytorch broadcasting work?


Instruction: Difference between PyTorch and exact analytical expression

Input:
While using Multivariate normal distribution in PyTorch I decided to compare it with exact analytical expression.
To my surprise, there was a small difference between them.
Is there any reason for this behaviour?
Firstly, calculate probabilities using MultivariateNormal:
1 from torch.distributions.multivariate_normal import MultivariateNormal
2 import torch
3 sigma = 2
4 m = MultivariateNormal(torch.zeros(2, dtype=torch.float32), torch.eye(2, dtype=torch.float32)*sigma**2)
5 values_temp = torch.zeros(size=(1,2), dtype=torch.float32)
6 out_torch = torch.exp(m.log_prob(values_temp))
7 out_torch
Out: tensor([0.0398])

Secondly, one can write exact formula for this case:
1 import numpy as np
2 out_exact = 1/(2*np.pi*sigma**2) * torch.exp(-torch.pow(values_temp, 2).sum(dim=-1)/(2*sigma**2))
3 out_exact
Out: tensor([0.0398])

There is a difference between them:
1 (out_torch - out_exact).sum()
Out: tensor(3.7253e-09)

Can someone help me understand the behavior of these two snippets? Which of these two expressions is more precise? Maybe someone can underline my mistake in any part of the code?

Output:
Most modern systems use the IEEE 754 standard to represent fixed precision floating point values. Because of this, we can be sure that neither the result provided from pytorch or the "exact" value you've computed are actually exactly equal to the analytical expression. We know this because the actual value of the expression is certainly irrational, and IEEE 754 cannot exactly represent any irrational number. This is a general phenomenon when using fixed precision floating point representations, which you can read more about on Wikipedia and this question.
Upon further analysis we find that the normalized difference you're seeing is on the order of machine epsilon (i.e. 3.7253e-09 / 0.0398 is approximately equal to torch.finfo(torch.float32).eps) indicating that the difference is likely just a result of inaccuracies of floating point arithmetic.
For a further demonstration we can write a mathematically equivalent expression to the one you have as
out_exact = torch.exp(np.log(1/ (2*np.pi*sigma**2)) + (-torch.pow(values_temp, 2).sum(dim=-1)/2/sigma**2))

which agrees exactly with value given by my current installation of pytorch.


Instruction: Arbitrary shaped Feedforward Neural Network in Pytorch

Input:
I am making a script that has some generative aspect to it, and I need to generate arbitrary shaped  feedforward NNs. The idea is to pass a list With the number of number of neurons in each layer, and the number of layers is determined by the length of the list:
shape = [784,64,64,64,10]

I tried something like this:
shapenn = [784,64,64,64,10]
class Net(nn.Module):
    def __init__(self, shapenn):
        super().__init__()
        self.shapenn = shapenn
        self.fcl = [] # list with fully conected leyers
        for i in range(len(act_funs)):
            self.fcl.append(nn.Linear(self.nnarch[i],self.nnarch[i+1]))
net = Net(shapenn)

While the fully connected layers are created correctly in the list fcl, net is not initialized properly for example it has not net.parameters().
I am sure there is a correct way to do this, thank you very much in advance.

Output:
You need to use nn.ModuleList in place of the built-in python list (similarly use nn.ModuleDict in place of python dictionaries). These behave like a normal list except that they must only contain instances that subclasses nn.Module and using them signals that the modules contained in the list should be considered submodules of your module. For example
import torch.nn as nn

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.fcl = nn.ModuleList()
        for i in range(5):
            self.fcl.append(nn.Linear(10, 10))

net = Net()
print([name for name, val in net.named_parameters()])

prints
['fcl.0.weight', 'fcl.0.bias', 'fcl.1.weight', 'fcl.1.bias', 'fcl.2.weight', 'fcl.2.bias', 'fcl.3.weight', 'fcl.3.bias', 'fcl.4.weight', 'fcl.4.bias']


Instruction: (image, mask) pair do not match one another in a semantic segmentation task

Input:
I am writing a simple custom DataLoader (which I will add more features to later) for a segmentation dataset but the (image, mask) pair I return using __getitem()__ method are different; the returned mask belongs to a different image than the one which is returned. My directory structure is /home/bohare/data/images and /home/bohare/data/masks .
Following is the code I have:
import torch
from torch.utils.data.dataset import Dataset
from PIL import Image
import glob
import os
import matplotlib.pyplot as plt

class CustomDataset(Dataset):
    def __init__(self, folder_path):

        self.img_files = glob.glob(os.path.join(folder_path,'images','*.png'))
        self.mask_files = glob.glob(os.path.join(folder_path,'masks','*.png'))

    def __getitem__(self, index):

        image = Image.open(self.img_files[index])
        mask = Image.open(self.mask_files[index])

        return image, mask

    def __len__(self):
        return len(self.img_files)

data = CustomDataset(folder_path = '/home/bohare/data')
len(data)

This code correctly gives out the total size of the dataset.
But when I use:
img, msk = data.__getitem__(n) where n is the index of any (image, mask) pair and I plot the image and mask, they do not correspond to one another.
How can I modify/what can I add to the code to make sure the (image, mask) pair are returned correctly? Thanks for the help.

Output:
glob.glob is returing it without order, glob.glob calls internally os.listdir:

os.listdir(path) Return a list containing the names of the entries in
the directory given by path. The list is in arbitrary order. It does
not include the special entries '.' and '..' even if they are present
in the directory.

To solve it, you can just sort both so that the order will be the same:
self.img_files = sorted(glob.glob(os.path.join(folder_path,'images','*.png')))
self.mask_files = sorted(glob.glob(os.path.join(folder_path,'masks','*.png')))


Instruction: Trying to understand what "save_for_backward" is in Pytorch

Input:
I have some knowledge in Pytorch,but i don't really understand the mechanisms of classes within Pytorch.
For example in the link: https://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_custom_function.html
you can find the following code:
import torch

class MyReLU(torch.autograd.Function):

    @staticmethod
    def forward(ctx, input):
        ctx.save_for_backward(input)
        return input.clamp(min=0)

    @staticmethod
    def backward(ctx, grad_output):
        input, = ctx.saved_tensors
        grad_input = grad_output.clone()
        grad_input[input < 0] = 0
        return grad_input

i am only focusing on the forward method of this class,and i am wondering what
ctx.save_for_backward(input)

does.Whether the previous line of code is present or not is irrelevent when i try the forward method on a concrete example:
a=torch.eye(3)
rel=MyReLU()
print(rel.forward(rel,a))

as i get the same result in both cases.Could someone explain me what is happening and why it is useful to add the save_for_backward?
Thank you in advance.

Output: The ctx.save_for_backward method is used to store values generated during forward() that will be needed later when performing backward(). The saved values can be accessed during backward() from the ctx.saved_tensors attribute.


Instruction: "view_as_windows" from skimage but in Pytorch

Input: Is there any Pytorch version of view_as_windows from skimage? I want to create the view while the tensor is on the GPU.

Output:
I needed the same functionality from Pytorch and ended up implementing it myself:
def view_as_windows_torch(image, shape, stride=None):
    """View tensor as overlapping rectangular windows, with a given stride.

    Parameters
    ----------
    image : `~torch.Tensor`
        4D image tensor, with the last two dimensions
        being the image dimensions
    shape : tuple of int
        Shape of the window.
    stride : tuple of int
        Stride of the windows. By default it is half of the window size.

    Returns
    -------
    windows : `~torch.Tensor`
        Tensor of overlapping windows

    """
    if stride is None:
        stride = shape[0] // 2, shape[1] // 2

    windows = image.unfold(2, shape[0], stride[0])
    return windows.unfold(3, shape[1], stride[1])

Essentially it is just two lines of Pytorch code relying on torch.Tensor.unfold. You can easily convince yourself, that it does the same as skimage.util.view_as_windows:
import torch

x = torch.arange(16).reshape((1, 1, 4, 4))

patches = view_as_windows_torch(image=x, shape=(2, 2))
print(patches)

Gives:
tensor([[[[[[ 0,  1],
            [ 4,  5]],

           [[ 1,  2],
            [ 5,  6]],

           [[ 2,  3],
            [ 6,  7]]],

          [[[ 4,  5],
            [ 8,  9]],

           [[ 5,  6],
            [ 9, 10]],

           [[ 6,  7],
            [10, 11]]],

          [[[ 8,  9],
            [12, 13]],

           [[ 9, 10],
            [13, 14]],

           [[10, 11],
            [14, 15]]]]]])

I hope this helps!


Instruction: Convert from Keras to Pytorch - conv2d

Input:
I am trying to convert the following Keras code into PyTorch.
    tf.keras.Sequential([
          Conv2D(128, 1, activation=tf.nn.relu),
          Conv2D(self.channel_n, 1, activation=None),
    ])

When creating the model summary with self.channels=16 i get the following summary.
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
conv2d (Conv2D)              (1, 3, 3, 128)            6272
_________________________________________________________________
conv2d_1 (Conv2D)            (1, 3, 3, 16)             2064
=================================================================
Total params: 8,336
Trainable params: 8,336
Non-trainable params: 0

How would one convert?
I have attempted it as such:
import torch
from torch import nn

class CellCA(nn.Module):
    def __init__(self, channels, dim=128):
        super().__init__()
        self.net = nn.Sequential(
            nn.Conv2d(in_channels=channels,out_channels=dim, kernel_size=1),
            nn.ReLU(),
            nn.Conv2d(in_channels=dim, out_channels=channels, kernel_size=1),
        )
    def forward(self, x):
        return self.net(x)

However, I get 4240 params

Output: The attempt above is correct if you configure the initial channels in correcty (48 in this case).


Instruction: How do I make sure Vulkan is using the same GPU as CUDA?

Input:
I'm using an application that uses both vulkan and cuda (specifically pytorch) on an HPC cluster (univa grid engine).
When a job is submitted, the cluster scheduler sets an environment variable SGE_HGR_gpu which contains a GPU ID for the job to use (so other jobs run by other users do not use the same GPU)
The typical way to tell an application that uses CUDA to use a specific GPU is to set CUDA_VISIBLE_DEVICES=n
As i'm also using Vulkan, I dont know how to make sure that I choose the same device from those that are listed with vkEnumeratePhysicalDevices.
I think that the order of the values that 'n' can take is the same as the order of the devices on the PCI BUS, however I dont know if the order of the devices returned by vkEnumeratePhysicalDevices are in this order, and the documentation does not specify what this order is.
So how can I go about making sure i'm choosing the same physical GPU for both Vulkan and CUDA?

Output:
With VkPhysicalDeviceIDPropertiesKHR (Vulkan 1.1) resp VkPhysicalDeviceVulkan11Properties (Vulkan 1.2) you can get device UUID, which is one of the formats CUDA_VISIBLE_DEVICES seems to use. You should also be able to convert index to UUID (or vice versa) with nvidia-smi -L (or with NVML library).
Or other way around, cudaDeviceProp includes PCI info which could be compared to VK_EXT_pci_bus_info extensions output.
If the order matches in Vulkan, it is best to ask NVidia directly; I cannot find info how NV orders them. IIRC from the Vulkan Loader implementation, the order should match the order from registry, and then the order the NV driver itself gives them. Even so you would have to filter non-NV GPUs from the list in generic code, and you do not know if the NV Vulkan ICD implementation matches CUDA without asking NV.


Instruction: Pytorch CrossEntropyLoss from single dimensional Tensors

Input:
I am missing something here: why does CrossEntropyLoss not work with single dimensional Tensors?
from  torch  import Tensor
X =Tensor([1.0,2.0,3.0])
labs = Tensor([2,2,3])
loss = nn.CrossEntropyLoss().forward(X,labs)

_stacklevel, dtype)
   1315         dim = _get_softmax_dim('log_softmax', input.dim(), _stacklevel)
   1316     if dtype is None:
-> 1317         ret = input.log_softmax(dim)
   1318     else:
   1319         ret = input.log_softmax(dim, dtype=dtype)

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

Why does that fail and what should be changed to get the desired result?

Output:
if you see the documentation here!
Input: (N,C) where C = number of classes
Target: (N) where each value is 0 <= targets[i] <= C-1
Output: scalar. If reduce is False, then (N) instead.

So it expects the input as 2D tensor
and target as 1D
import torch
from  torch  import Tensor
X =Tensor([[1.0,2.0,3.0]])   #2D
labs = torch.LongTensor([2]) # 0 <= targets[i] <= C-1
loss = nn.CrossEntropyLoss().forward(X,labs)


Instruction: (Efficiently) expanding a feature mask tensor to match embedding dimensions

Input:
I have a B (batch size) by F (feature count) mask tensor M, which I'd like to apply (element-wise multiply) to an input x.
...Thing is, my x has had its raw feature columns transformed into embeddings of non-constant width, so its total dimensions are B by E (embedded size).
My draft code is along the lines of:
# Given something like:
M = torch.Tensor([[0.2, 0.8], [0.5, 0.5], [0.6, 0.4]])  # B=3, F=2
x = torch.Tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 0], [11, 12, 13, 14, 15]])  # E=5

feature_sizes = [2, 3]  # (Feature 0 embedded to 2 cols, feature 1 to 3)

# In forward() pass:
components = []
for ix, size in enumerate(feature_sizes):
    components.append(M[:, ix].view(-1, 1).expand(-1, size))
M_x = torch.cat(components, dim=1)

# Now M_x is (B, E) and can be mapped with x

> M_x = torch.Tensor([
>     [0.2, 0.4, 2.4, 3.6, 4],
>     [3, 3.5, 4, 4.5, 0],
>     [6.6, 7.2, 5.2, 5.6, 6],
> ])

My question is: are there any obvious optimizations I'm missing here? Is that for loop the right way to go about it, or is there some more direct way to achieve this?
I have control over the embedding process, so can store whatever representation is helpful and am e.g. not tied to feature_sizes list of ints.

Output:
Duh, I forgot: An indexing operation could do this!
Given the above situation (but I'll take a more complex feature_sizes to show a bit more clearly), we can pre-compute an index tensor with something like:
# Given that:
feature_sizes = [1, 3, 1, 2]

# Produce nested list e.g. [[0], [1, 1, 1], [2], [3, 3]]:
ixs_per_feature = [[ix] * size for ix, size in enumerate(feature_sizes)]

# Flatten out into a vector e.g. [0, 1, 1, 1, 2, 3, 3]:
mask_ixs = torch.LongTensor(
    [item for sublist in ixs_per_feature for item in sublist]
)

# Now can directly produce M_x by indexing M:
M_x = M[:, mask_ixs]

I got a modest speedup by using this method instead of the for loops.


Instruction: Same padding when kernel size is even

Input:
When the kernel size is odd, we can manually calculate the necessary padding to get the output in the same dimension as input such that it creates same padding.
But how can we calculate padding dimensions for kernels with even sizes (ex: (2x2)?

Output: I used padding as 1 and dilation as 2 which resulted same padding.


Instruction: Is there an analogy for Python's array slicing in C++ (libtorch)?

Input:
In Python with PyTorch, if you have an array:
torch.linspace(0, 10, 10)
you can use e.g. only the first three elements by saying
reduced_tensor = torch.linspace(0, 10, 10)[:4].
Is there an analog to the [:] array slicing in C++/libtorch? If not, how can I achieve this easily?

Output:
Yes, You can use  Slice  and  index  in libtorch. You can do:
auto tensor = torch::linspace(0, 10, 10).index({ Slice(None, 4) });

You can read more about indexing here.
Basically as its indicated in the documentation :

The main difference is that, instead of using the []-operator similar
to the Python API syntax, in the C++ API the indexing methods are:
torch::Tensor::index (link)
torch::Tensor::index_put_ (link)
It’s also important to note that index types such as None / Ellipsis /
Slice live in the torch::indexing namespace, and it’s recommended to
put using namespace torch::indexing before any indexing code for
convenient use of those index types.

For the convenience here is some of the Python vs C++ conversions taken from the link I just gave:
Here are some examples of translating Python indexing code to C++:
Getter
------

+----------------------------------------------------------+--------------------------------------------------------------------------------------+
| Python                                                   | C++  (assuming  using namespace torch::indexing )                                    |
+==========================================================+======================================================================================+
|  tensor[None]                                            |  tensor.index({None})                                                                |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[Ellipsis, ...]                                   |  tensor.index({Ellipsis, "..."})                                                     |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[1, 2]                                            |  tensor.index({1, 2})                                                                |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[True, False]                                     |  tensor.index({true, false})                                                         |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[1::2]                                            |  tensor.index({Slice(1, None, 2)})                                                   |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[torch.tensor([1, 2])]                            |  tensor.index({torch::tensor({1, 2})})                                               |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+
|  tensor[..., 0, True, 1::2, torch.tensor([1, 2])]        |  tensor.index({"...", 0, true, Slice(1, None, 2), torch::tensor({1, 2})})            |
+----------------------------------------------------------+--------------------------------------------------------------------------------------+

Translating between Python/C++ index types
------------------------------------------

The one-to-one translation between Python and C++ index types is as follows:

+-------------------------+------------------------------------------------------------------------+
| Python                  | C++ (assuming  using namespace torch::indexing )                       |
+=========================+========================================================================+
|  None                   |  None                                                                  |
+-------------------------+------------------------------------------------------------------------+
|  Ellipsis               |  Ellipsis                                                              |
+-------------------------+------------------------------------------------------------------------+
|  ...                    |  "..."                                                                 |
+-------------------------+------------------------------------------------------------------------+
|  123                    |  123                                                                   |
+-------------------------+------------------------------------------------------------------------+
|  True                   |  true                                                                  |
+-------------------------+------------------------------------------------------------------------+
|  False                  |  false                                                                 |
+-------------------------+------------------------------------------------------------------------+
|  :  or  ::              |  Slice()  or  Slice(None, None)  or  Slice(None, None, None)           |
+-------------------------+------------------------------------------------------------------------+
|  1:  or  1::            |  Slice(1, None)  or  Slice(1, None, None)                              |
+-------------------------+------------------------------------------------------------------------+
|  :3  or  :3:            |  Slice(None, 3)  or  Slice(None, 3, None)                              |
+-------------------------+------------------------------------------------------------------------+
|  ::2                    |  Slice(None, None, 2)                                                  |
+-------------------------+------------------------------------------------------------------------+
|  1:3                    |  Slice(1, 3)                                                           |
+-------------------------+------------------------------------------------------------------------+
|  1::2                   |  Slice(1, None, 2)                                                     |
+-------------------------+------------------------------------------------------------------------+
|  :3:2                   |  Slice(None, 3, 2)                                                     |
+-------------------------+------------------------------------------------------------------------+
|  1:3:2                  |  Slice(1, 3, 2)                                                        |
+-------------------------+------------------------------------------------------------------------+
|  torch.tensor([1, 2])   |  torch::tensor({1, 2})                                                 |
+-------------------------+------------------------------------------------------------------------+


Instruction: No module named 'parse_config' while tryhing to load checkpoint in PyTorch

Input:
I have a checkpoint file saved after training a model in Pytorch. I have to inspect it in a different module so I tried to load the checkpoint using the following code.
map_location = lambda storage, loc: storage
checkpoint = torch.load("model.pt", map_location=map_location)

But it is raising ModuleNotFoundError issue, which I couldn't find a way to resolve.
The error traceback  :
Traceback (most recent call last):
  File "main.py", line 11, in <module>
    model = loadmodel(hook_feature)
  File "/home/../model_loader.py", line 21, in loadmodel
    checkpoint = torch.load(settings.MODEL_FILE, map_location=map_location)
  File "/home/../.conda/envs/envreporting/lib/python3.6/site-packages/torch/serialization.py", line 584, in load
    return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
  File "/home/../.conda/envs/envreporting/lib/python3.6/site-packages/torch/serialization.py", line 842, in _load
    result = unpickler.load()
ModuleNotFoundError: No module named 'parse_config'

I couldn't find an already existing issue relevant to this one.

Output: Is it possible that you have used https://github.com/victoresque/pytorch-template for training the model ? In that case, the project also saves its config in the checkpoint and you also need to import parse_config.py file in order to load it.


Instruction: Use pre trained Nodes from past runs - Pytorch Biggraph

Input:
After struggling with this amazing facebookresearch
/
PyTorch-BigGraph project, and its impossible API, I managed to get a grip on how to run it (thanks to stand alone simple example)
My system restrictions do not allow me to train the dense (embedding) representation of all edges, and I need from time to time to upload past embeddings and train the model using both new edges and existing nodes, notice that nodes in past and new edge list do not necessarily overlap.
I tried to understand from here: see the context section how to do it, so far with no success.
Following is a stand-alone PGD code, that turned batch_edges into an embedding node list, however, I need it to use pre-trained nodes list past_trained_nodes.
import os
import shutil
from pathlib import Path

from torchbiggraph.config import parse_config
from torchbiggraph.converters.importers import TSVEdgelistReader, convert_input_data
from torchbiggraph.train import train
from torchbiggraph.util import SubprocessInitializer, setup_logging

DIMENSION = 4
DATA_DIR = 'data'
GRAPH_PATH = DATA_DIR + '/output1.tsv'
MODEL_DIR = 'model'

raw_config = dict(
        entity_path=DATA_DIR,
        edge_paths=[DATA_DIR + '/edges_partitioned', ],
        checkpoint_path=MODEL_DIR,
        entities={"n": {"num_partitions": 1}},
        relations=[{"name": "doesnt_matter", "lhs": "n", "rhs": "n", "operator": "complex_diagonal", }],
        dynamic_relations=False, dimension=DIMENSION, global_emb=False, comparator="dot",
        num_epochs=7, num_uniform_negs=1000, loss_fn="softmax", lr=0.1, eval_fraction=0.,)

batch_edges = [["A", "B"], ["B", "C"],  ["C", "D"], ["D", "B"], ["B", "D"]]

# I want the model to use these pretrained nodes, Notice that Node A exist, And F Does not
#I dont have all past nodes, as some are gained from data
past_trained_nodes = {'A': [0.5, 0.3, 1.5, 8.1], 'F': [3, 0.6, 1.2, 4.3]}

try:
    shutil.rmtree('data')
except:
    pass
try:
    shutil.rmtree(MODEL_DIR)
except:
    pass

os.makedirs(DATA_DIR, exist_ok=True)
with open(GRAPH_PATH, 'w') as f:
    for edge in batch_edges:
        f.write('\t'.join(edge) + '\n')

setup_logging()
config = parse_config(raw_config)
subprocess_init = SubprocessInitializer()
input_edge_paths = [Path(GRAPH_PATH)]

convert_input_data(config.entities, config.relations, config.entity_path, config.edge_paths,
                   input_edge_paths, TSVEdgelistReader(lhs_col=0, rel_col=None, rhs_col=1),
                   dynamic_relations=config.dynamic_relations, )

train(config, subprocess_init=subprocess_init)

How can I use my pre-trained nodes in the current model?
Thanks in advance!

Output:
Since torchbiggraph is file based, you can modify the saved files to load pre-trained embeddings and add new nodes. I wrote a function to achieve this
import json

def pretrained_and_new_nodes(pretrained_nodes,new_nodes,entity_name,data_dir,embeddings_path):
    """
    pretrained_nodes:
        A dictionary of nodes and their embeddings
    new_nodes:
        A list of new nodes,each new node must have an embedding in pretrained_nodes.
        If no new nodes, use []
    entity_name:
        The entity's name, for example, WHATEVER_0
    data_dir:
        The path to the files that record graph nodes and edges
    embeddings_path:
        The path to the .h5 file of embeddings
    """
    with open('%s/entity_names_%s.json' % (data_dir,entity_name),'r') as source:
        nodes = json.load(source)
    dist = {item:ind for ind,item in enumerate(nodes)}

    if len(new_nodes) > 0:
        # modify both the node names and the node count
        extended = nodes.copy()
        extended.extend(new_nodes)
        with open('%s/entity_names_%s.json' % (data_dir,entity_name),'w') as source:
            json.dump(extended,source)
        with open('%s/entity_count_%s.txt' % (data_dir,entity_name),'w') as source:
            source.write('%i' % len(extended))

    if len(new_nodes) == 0:
        # if no new nodes are added, we won't bother create a new .h5 file, but just modify the original one
        with h5py.File(embeddings_path,'r+') as source:

            for node,embedding in pretrained_nodes.items():
                if node in nodes:
                    source['embeddings'][dist[node]] = embedding
    else:
        # if there are new nodes, then we must create a new .h5 file
        # see https://stackoverflow.com/a/47074545/8366805
        with h5py.File(embeddings_path,'r+') as source:
            embeddings = list(source['embeddings'])
            optimizer = list(source['optimizer'])
        for node,embedding in pretrained_nodes.items():
            if node in nodes:
                embeddings[dist[node]] = embedding
        # append new nodes in order
        for node in new_nodes:
            if node not in list(pretrained_nodes.keys()):
                raise ValueError
            else:
                embeddings.append(pretrained_nodes[node])
        # write a new .h5 file for the embedding
        with h5py.File(embeddings_path,'w') as source:
            source.create_dataset('embeddings',data=embeddings,)
            optimizer = [item.encode('ascii') for item in optimizer]
            source.create_dataset('optimizer',data=optimizer)

After you trained a model (let's say the stand along simple example you linked in your post), and you want to change the learned embedding of node A to [0.5, 0.3, 1.5, 8.1]. Moreover, you also want to add a new node F to the graph with embedding [3, 0.6, 1.2, 4.3] (This newly added node F has no connections with other nodes). You can run my function with
past_trained_nodes = {'A': [0.5, 0.3, 1.5, 8.1], 'F': [3, 0.6, 1.2, 4.3]}
pretrained_and_new_nodes(pretrained_nodes=past_trained_nodes,
                         new_nodes=['F'],
                         entity_name='WHATEVER_0',
                         data_dir='data/example_1',
                         embeddings_path='model_1/embeddings_WHATEVER_0.v7.h5')

After you ran this function, you can check the modified file of embeddings embeddings_WHATEVER_0.v7.h5
filename = "model_1/embeddings_WHATEVER_0.v7.h5"

with h5py.File(filename, "r") as source:
    embeddings = list(source['embeddings'])

embeddings

and you will see, the embedding of A is changed, and also the embedding of F is added (the order of the embeddings is consistent with the order of nodes in entity_names_WHATEVER_0.json).
With the files modified, you can use the pre-trained nodes in a new training session.


Instruction: torch.cuda.is_available() True in (base), but False in other conda env

Input:
I run this code in Anaconda prompt, and it returns True.
(base) C:\User
torch.cuda.is_available()
True

But when I run other conda environment, it just doesn't work.
(pytorch_project) C:\User
torch.cuda.is_available()
False

The problem seems to be different results of torch.version.cuda.
(base) torch.version.cuda = 10.1
(pytorch_project) torch.version.cuda = 10.2
But I don't know why they are different...
How do I make 10.2 down to 10.1 and make is_available() == True ?

Here is my info.
 Windows 10 / nvidia-smi=425.31 / CUDA ver=10.1 / pytorch=1.4.0 / torchvision=0.5.0

conda list

Output:
Thank you for your answers and comments. <3
I've solved the problem.
I use Visual Studio Code as the developer environment tool, but as shown in the picture I uploaded, conda list pointed at exact the same directory, which means I didn't actually activate my environment.
It should be like (pytorch) C:\User , but it's (Power Shell) PS C:\User instead.
To solve this, I went to Settings → Terminal > Integrated > Shell Args: Windows and edited settings.json with
"terminal.integrated.shellArgs.windows": ["-ExecutionPolicy", "ByPass", "-NoExit","-Command","& 'C:\\Users\\user\\miniconda3\\shell\\condabin\\conda-hook.ps1'" ]

Reference: https://blog.lcarbon.idv.tw/vscode-設定-anaconda-路徑至-visual-studio-code-終端機中windows/
By starting a new terminal I got (pytorch) PS C:\User correctly, and saw my torch=1.5.0 using conda list.
Then I run conda install pytorch torchvision cudatoolkit=10.1 -c pytorch to change the pytorch version.
And Voilà!!
torch.cuda.is_available()
True


Instruction: Directly update the optimizer learning rate

Input:
I have a specific learning rate schedule in mind. It is based on the epoch but differs from the generally available ones I am aware of including StepLR.
Is there something that would perform the equivalent to:
optimizer.set_lr(lr)
or
optimizer.set_param('lr,',lr)
I would then simply invoke that method at the end of each epoch (or possibly even more frequently)
Context: I am using the adam optimizer as so:
    optimizer = torch.optim.Adam(model.parameters(), lr=LrMax, weight_decay=decay) # , betas=(args.beta1, args.beta2)

Update  I found this information https://discuss.pytorch.org/t/change-learning-rate-in-pytorch/14653:
for param_group in optimizer.param_groups:
        param_group['lr'] = lr

Is there a way to ascertain that the adam optimizer being used is employing the new learning rate?

Output:
You can do this in this way:
for param_group in optimizer.param_groups:
    param_group['lr'] = lr


Instruction: Neural Network : Epoch and Batch Size

Input:
I am trying to train a neural network to classify words into different categories.
I notice two things:
When I use a smaller batch_size (like 8,16,32) the loss is not decreasing, but rather sporadically varying. When I use a larger batch_size (like 128, 256), the loss is going going down, but very slowly.
More importantly, when I use a larger EPOCH value, my model does a good job at reducing the loss. However I'm using a really large value (EPOCHS = 10000).
Question:
How to get the optimal EPOCH and batch_size values?

Output:
There is no way to decide on these values based on some rules. Unfortunately, the best choices depend on the problem and the task. However, I can give you some insights.
When you train a network, you calculate a gradient which would reduce the loss. In order to do that, you need to backpropagate the loss. Now, ideally, you compute the loss based on all of the samples in your data because then you consider basically every sample and you come up with a gradient that would capture all of your samples. In practice, this is not possible due to the computational complexity of calculating gradient on all samples. Because for every update, you have to compute forward-pass for all your samples. That case would be batch_size = N, where N is the total number of data points you have.
Therefore, we use small batch_size as an approximation! The idea is instead of considering all the samples, we say I compute the gradient based on some small set of samples but the thing is I am losing information regarding the gradient.
Rule of thumb:
Smaller batch sizes give noise gradients but they converge faster because per epoch you have more updates. If your batch size is 1 you will have N updates per epoch. If it is N, you will only have 1 update per epoch. On the other hand, larger batch sizes give a more informative gradient but they convergence slower.
That is the reason why for smaller batch sizes, you observe varying losses because the gradient is noisy. And for larger batch sizes, your gradient is informative but you need a lot of epochs since you update less frequently.
The ideal batch size should be the one that gives you informative gradients but also small enough so that you can train the network efficiently. You can only find it by trying actually.


Instruction: How to writte a raster plot?

Input:
I'm using pytorch on google colab.
I've got a tensor matrix below, this is the example, and actually the matrix size is about 50 neurons and 30,000~50,000 time.
a= torch.tensor([[0., 0., 0., 0., 0.],
                 [0., 0., 0., 0., 1.],
                 [0., 1., 0., 1., 0.]])

each of values are,
a= torch.tensor([[Neuron1(t=1), N2(t=1), N3(t=1), N4(t=1), N5(t=1)],
                 [N1(t=2), N2(t=2), N3(t=2), N4(t=2), N5(t=2)],
                 [N1(t=3), N2(t=3), N3(t=3), N4(t=3), N5(t=3)]])

and 1 means that neuron fire, 0 means not fire.
So Neuron5(t=2), Neuron2(t=3) and Neuron4(t=3) are firing.
Then, I want to make a raster plot or scatter plot like below using this matrix,
The dots show the firing neuron.

neuron number
1|
2|          *
3|
4|          *
5|__ *_____time
    1  2  3
What would be the best python code to do this?
I have no idea now.
Thank you for reading.

Output:
You can do it easily as follows:
import matplotlib.pyplot as plt
a= torch.tensor([[0., 0., 0., 0., 0.],
                 [0., 0., 0., 0., 1.],
                 [0., 1., 0., 1., 0.]],device='cuda')
plt.scatter(*torch.where(a.cpu()))


Instruction: One instance with multple GPUs or multiple instances with one GPU

Input:
I am running multiple models using GPUs and all jobs combined can be run on 4 GPUs, for example. Multiple jobs can be run on the same GPU since the GPU memory can handle it.
Is it a better idea to spin up a powerful instance with all 4 GPUs as part of it and run all the jobs on one instance? Or should I go the route of having multiple instances with 1 GPU on each?
There are a few factors I'm thinking of:

Latency of reading files. Having a local disk on one machine should be faster latency wise, but it would be a quite a few reads from one source. Would this cause any issues?
I would need quite a few vCPU and a lot of memory to scale the IOPS since GPC scales IOPS that way, apparently. What is the best way to approach this? If anyone has any more on this, would appreciate pointers.
If in the future I need to downgrade to save costs/downgrade performance, I could simple stop the instance and change my specs.
Having everything on one machine would be easier to work with. I know in production I would want a more distributed approach, but this is strictly experimentation.

Those are my main thoughts. Am I missing something? Thanks for all of the help.

Output: Ended up going with one machine with multiple GPUs. Just assigned the jobs to the different GPUs to make the memory work.


Instruction: How to use a numpy-based external library function within a Tensorflow 2/keras custom layer?

Input:
I would be very grateful if someone would help me understanding the following problem.
I am trying to implement a custom layer in tensorflow 2 by using keras (it's a layer derived from the class Layer).
I have overriden the build and the call functions.
When I write my call function I need to invoke a method from an external library which only accepts numpy array. It's a pretty complex function and of course it does not use tensorflow functions.
My call function takes in input a tensor, convert it to numpy (by .numpy() function), invoke the external method from another library, then converts the numpy array back to tensor.
Here I have some issues and tried different solutions.

If I run the code it tells me that ...ops.Tensor has no method .numpy(). If I correctly understood, this is due to the type of tensor I receive in input to the call function and I need to use the eager mode.
If I compile by setting the run_eagerly flag to true, during the fit, it tells me that the gradient is missing. I was hoping that some form of auto-grad computation was implemented but maybe this is not what it is currently happening?.

Source code:
class SimpleLayer(tf.keras.layers.Layer):
    def __init__(self):
        super(SimpleLayer, self).__init__()

    def build(self, input_shape):
        shape_a = input_shape[0]                #batch_size
        shape_b = int(input_shape[1])           #height
        shape_c = int(input_shape[2])           #width
        shape_d = input_shape[3]                #channel_number
        self.w = self.add_weight(shape=(shape_d*shape_a, 2, shape_b, shape_c),
                                 initializer='random_uniform', dtype='float64',
                                 trainable=True)

    def call(self, inputs):
        tf_inputs = inputs.numpy()
        sft = tr.transform(tf_inputs) # call to the external library
        transformed = tf.math.multiply(sft, self.w)
        conv = tf.zeros(np.shape(inputs))
        conv = tr.inv_transform(inputs, transformed) # another call to a function that also converts to tensor
        return conv. # this is a tensor!

.....

model = tf.keras.Sequential([
    layers.experimental.preprocessing.Rescaling(1./255),
    pl.SimpleLayer(),
    layers.Activation('relu'),
    layers.MaxPooling2D(),
    pl.SimpleLayer(),
    layers.Activation('relu'),
    layers.Flatten(),
    layers.Dense(128, activation='relu'),
    layers.Dense(num_classes)
])

model.compile(
    optimizer='adam',
    loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),
    run_eagerly=True,  # adding this to avoid graph building
    metrics=['accuracy'])

history = model.fit(train_images, train_labels, epochs=3, batch_size=20,
                    validation_data=(test_images, test_labels))

Is there any workaround for that? Can I only use tensorflow 2 functions to process the tensor or maybe I am missing something?
I am also considering to switch to pyTorch (which I do not know yet) but I fear to have the same issue. What would you suggest?

Output:
Gradient cannot flow through numpy functions as... numpy has no autograd capabilities of any sort.
Your custom function will not work in either tensorflow2.x nor in pytorch as the operation on numpy arrays are not recorded.
For your call code (in PyTorch it would be forward, otherwise really similar), see comments:
def call(self, inputs):
    tf_inputs = inputs.numpy() # gradient breaks
    sft = tr.transform(tf_inputs) # still numpy
    transformed = tf.math.multiply(sft, self.w)
    conv = tf.zeros(np.shape(inputs))
    conv = tr.inv_transform(inputs, transformed)
    return conv # Tensor with no gradient history

And this would happen for all your SimpleLayer layers.
Possible solutions

Code those operations in differentiable manner (assuming they are differentiable and do not use operations like argmax) either in tensorflow or pytorch (the latter is easier and more intuitive and you would have easier time with it IMO).
If your complicated functions use numpy and are differentiable then you could code it with PyTorch by rewriting np.op to torch.op as their APIs are pretty similar and interoperate almost seamlessly
You may go with JAX instead of tf and pytorch as "JAX can automatically differentiate native Python and NumPy functions". Depending on your application it might be worth a shot


Instruction: Validate on entire validation set when using ddp backend with PyTorch Lightning

Input:
I'm training an image classification model with PyTorch Lightning and running on a machine with more than one GPU, so I use the recommended distributed backend for best performance ddp (DataDistributedParallel). This naturally splits up the dataset, so each GPU will only ever see one part of the data.
However, for validation, I would like to compute metrics like accuracy on the entire validation set and not just on a part. How would I do that? I found some hints in the official documentation, but they do not work as expected or are confusing to me. What's happening is that validation_epoch_end is called num_gpus times with 1/num_gpus of the validation data each. I would like to aggregate all results and only run the validation_epoch_end once.
In this section they state that when using dp/ddp2 you can add an additional function called like this
def validation_step(self, batch, batch_idx):
    loss, x, y, y_hat = self.step(batch)
    return {"val_loss": loss, 'y': y, 'y_hat': y_hat}

def validation_step_end(self, self, *args, **kwargs):
    # do something here, I'm not sure what,
    # as it gets called in ddp directly after validation_step with the exact same values
    return args[0]

However, the results are not being aggregated and validation_epoch_end is still called num_gpu times. Is this kind of behavior not available for ddp? Is there some other way how achieve this aggregation behavior?

Output:
training_epoch_end() and validation_epoch_end() receive data that is aggregated from all training / validation batches of the particular process. They simply receive a list of what you returned in each training or validation step.
When using the DDP backend, there's a separate process running for every GPU. There's no simple way to access the data that another process is processing, but there's a mechanism for synchronizing a particular tensor between the processes.
The easiest approach for computing a metric on the entire validation set is to calculate the metric in pieces and then synchronize the resulting tensor, for example by taking the average. self.log() calls will automatically synchronize the value between GPUs when you use sync_dist=True. How the value is synchronized is determined by the reduce_fx argument, which by default is torch.mean.
If you're happy with averaging the metric over batches too, you don't need to override training_epoch_end() or validation_epoch_end() — self.log() will do the averaging for you.
If the metric cannot be calculated separately for each GPU and then averaged, it can get a bit more challenging. It's possible to update some state variables at each step, and then synchronize the state variables at the end of an epoch and calculate the metric. The recommended way is to create a class that derives from the Metric class from the TorchMetrics project. Add the state variables in the constructor using add_state() and override the update() and compute() methods. The API will take care of synchronizing the state variables between the GPU processes.
There's already an accuracy metric in TorchMetrics and the source code is a good example of how to use the API.


Instruction: How to check which pytorch version fits torchvision

Input:
I am trying to clone and run this repository:
https://github.com/switchablenorms/CelebAMask-HQ
The demo runs with PyTorch 0.4.1, and I am trying to find the corresponding version, how can I find it?

Output:
The corresponding torchvision version for 0.4.1 is 0.2.1.
The easiest way is to look it up in the previous versions section. Only if you couldn't find it, you can have a look at the torchvision release data and pytorch's version. There you can find which version, got release with which version!


Instruction: how to Implement replication_pad2d layer in to coremltools converters as torch_op

Input:
I've been trying to convert my pytorch model into coreML format, However one of the layers is currently not supported replication_pad2d. Therefore I was trying to implement it using the register operator decorator @register_torch_op  to reimplement the layer for coremltools.converters, However I'm struggling to understand the input types to be able to implement the function currently. I got this, which is an implementation roughly translated from pytorch but it's not working
from coremltools.converters.mil import Builder as mb
from coremltools.converters.mil import register_torch_op
from coremltools.converters.mil.frontend.torch.ops import _get_inputs

@register_torch_op
def replication_pad2d(context, node):
  inputs = _get_inputs(context, node)
  x = inputs[0]
  a = len(x)
  L_list, R_list = [], []
  U_list, D_list = [], []
  for i in range(a):#i:0, 1
    l = x[:, :, :, (a-i):(a-i+1)]
    L_list.append(l)
    r = x[:, :, :, (i-a-1):(i-a)]
    R_list.append(r)
  L_list.append(x)
  x = mb.concat(L_list+R_list[::-1], axis=3, name=node.name)
  for i in range(a):
    u = x[:, :, (a-i):(a-i+1), :]
    U_list.append(u)
    d = x[:, :, (i-a-1):(i-a), :]
    D_list.append(d)
  U_list.append(x)
  x = mb.concat(U_list+D_list[::-1], axis=3, name=node.name)
  context.add(x)

but getting the following error
<ipython-input-12-cf14ed84cb93> in replication_pad2d(context, node)
     59   inputs = _get_inputs(context, node)
     60   x = inputs[0]
---> 61   a = len(x)
     62   L_list, R_list = [], []
     63   U_list, D_list = [], []

TypeError: object of type 'Var' has no len()

would be great if someone could help me understand this better especially input type node and context

Output:
I think the you can use the existing padding layer as:
from coremltools.converters.mil import Builder as mb
from coremltools.converters.mil import register_torch_op
from coremltools.converters.mil.frontend.torch.ops import _get_inputs

@register_torch_op(torch_alias=["replication_pad2d"])
def HackedReplication_pad2d(context, node):
    inputs = _get_inputs(context, node)
    x = inputs[0]
    pad = inputs[1].val
    x_pad = mb.pad(x=x, pad=[pad[2], pad[3], pad[0], pad[1]], mode='replicate')
    context.add(x_pad, node.name)

The documentation of the padding operation is not that great, so ordering of padding parameters is a guessing game.


Instruction: Defining named parameters for a customized NN module in Pytorch

Input:
This question is about how to appropriately define the parameters of a customized layer in Pytorch.  I am wondering how one can make the parameter to end up being named parameter?
class My_layer(torch.nn.Module):
    def __init__(self):
        self.mu = torch.nn.Parameter(torch.tensor([[0.0],[1.0]]))

So that when I print the parameters as below, the p.name is not empty.
my_layer = My_Layer()
for p in my_layer.parameters():
     print(p.name, p.data, p.requires_grad)

Output:
You are registering your parameter properly, but you should use nn.Module.named_parameters rather than nn.Module.parameters to access the names. Currently you are attempting to access Parameter.name, which is probably not what you want. The name attribute of Parameter and Tensor do not appear to be documented, but as far as I can tell, they refer to an internal attribute of the _C._TensorBase class and cannot be modified externally.
Every time you assign a Parameter to an attribute of your module it is registered with a name (this occurs in nn.Module.__setattr__ here). The parameter always takes the same name as the attribute itself, so "mu" in this case. To iterate over all the parameters and their associated names use nn.Module.named_parameters. For example,
my_layer = My_Layer()
for n, p in my_layer.named_parameters():
    print('Parameter name:', n)
    print(p.data)
    print('requires_grad:', p.requires_grad)

which prints
Parameter name: mu
tensor([[0.],
        [1.]])
requires_grad: True


Instruction: Pytorch, Pre-trained model: How to use feature and classifier at the same time

Input:
I'm using vgg16 extracting image feature vector. I want to get 114096 vector from the 2nd-to-last layer.

My code:
def get_model():
    model = models.vgg16(pretrained=True)#.features[:].classifier[:4]
    model = model.eval()
    # model.cuda()  # send the model to GPU, DO NOT include this line if you haven't a GPU
    return model

But I can only get 111000 vector from the last layer.
I know how to use feathers and classifier, but I don't know how to use them at the same time.
use classifier only:

use feathers only:

use them at the same time:

log:
Traceback (most recent call last):
  File "/mnt/c/Users/sunji/PycharmProjects/image_cluster_pytorch/main.py", line 7, in <module>
    model = calc.get_model()
  File "/mnt/c/Users/sunji/PycharmProjects/image_cluster_pytorch/imagecluster/calc.py", line 17, in get_model
    model = models.vgg16(pretrained=True).features[:].classifier[:4]
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 771, in __getattr__
    raise ModuleAttributeError("'{}' object has no attribute '{}'".format(
torch.nn.modules.module.ModuleAttributeError: 'Sequential' object has no attribute 'classifier'

Output:
I've find the solution myself.
Sorry for bothering to Stack overflow.
here is the code:
def get_model():
    model = models.vgg16(pretrained=True)
    model.features = model.features[:]
    model.classifier = model.classifier[:4]

    model = model.eval()
    # model.cuda()  # send the model to GPU, DO NOT include this line if you haven't a GPU
    return model

result:

I think this is the right answer.


Instruction: Inconsitencies/Inaccuracies between torchvision.datasets.MNIST and Michael Nielsens neuralnetworksanddeeplearning

Input:
I printed the vectorized form of the first training image of the mnist dataset of pytorch and https://github.com/mnielsen/neural-networks-and-deep-learning/tree/master/data .
The difference seems too big for just floating point precision error.
Full Diff of first mnist train image:
https://www.diffchecker.com/6y6YTFiN

Code to reproduce:
import mnist_loader
import numpy
numpy.set_printoptions(formatter={'float': '{: 0.4f}'.format})

d = mnist_loader.load_data()
print numpy.array(d[0][0][0]).reshape(784,1)

pytorch:
import torch
import torchvision
from torchvision import transforms as tf

dtype = torch.float32

transforms = tf.Compose([tf.ToTensor()])

mnist_data = torchvision.datasets.MNIST("./mnist", transform=transforms, download=True)
data_loader = torch.utils.data.DataLoader(mnist_data,
                                          batch_size=500,
                                          shuffle=False,
                                          num_workers=1)

data, target = next(data_loader.__iter__())
print(data[0].reshape(784,1))

Anyone an idea what's causing this ?

Output: MNIST images consist of pixel values that are integers in the range 0 to 255 (inclusive). To produce the tensors you are looking at, those integer values have been normalised to lie between 0.0 and 1,0, by dividing them all by some constant factor. It appears that your two sources chose different normalising factors: 255 in one case and 256 in the other.


Instruction: PyTorch not updating weights when using autograd in loss function

Input:
I am trying to use the gradient of a network with respect to its inputs as part of my loss function. However, whenever I try to calculate it, the training proceeds but the weights do not update
import torch
import torch.optim as optim
import torch.autograd as autograd

ic = torch.rand((25, 3))
ic = torch.tensor(ic, requires_grad=True)
optimizer = optim.RMSprop([ic], lr=1e-2)

for itr in range(1, 50):
    optimizer.zero_grad()
    sol = torch.tanh(.5*torch.stack(100*[ic])) # simplified for minimal working example

    dx = sol[-1, :, 0]
    dxdxy, = autograd.grad(dx,
                           inputs=ic,
                           grad_outputs = torch.ones(ic.shape[0]), # batchwise
                           retain_graph=True
                          )
    dxdxy = torch.tensor(dxdxy, requires_grad=True)
    loss = torch.sum(dxdxy)

    loss.backward()
    optimizer.step()

    if itr % 5 == 0:
        print(loss)

What am I doing wrong?

Output:
When you run autograd.grad without setting flag create_graph to True then you won't obtain an output which is connected to the computational graph, which means that you won't be able to further optimize w.r.t ic (and obtain a higher order derivative as you wish to do here).
From torch.autograd.grad's docstring:

create_graph (bool, optional): If True, graph of the derivative will
be constructed, allowing to compute higher order derivative products.
Default: False.

Using dxdxy = torch.tensor(dxdxy, requires_grad=True) as you've tried here won't help since the computational graph which is connected to ic has been lost by then (since create_graph is False), and all you do is create a new computational graph with dxdxy being a leaf node.
See the solution attached below (note that when you create ic you can set requires_grad=True and hence the second line is redundant (that's not a logical problem but just longer code):
import torch
import torch.optim as optim
import torch.autograd as autograd

ic = torch.rand((25, 3),requires_grad=True) #<-- requires_grad to True here
#ic = torch.tensor(ic, requires_grad=True) #<-- redundant
optimizer = optim.RMSprop([ic], lr=1e-2)

for itr in range(1, 50):
    optimizer.zero_grad()
    sol = torch.tanh(.5 * torch.stack(100 * [ic]))  # simplified for minimal working example

    dx = sol[-1, :, 0]
    dxdxy, = autograd.grad(dx,
                           inputs=ic,
                           grad_outputs=torch.ones(ic.shape[0]),  # batchwise
                           retain_graph=True, create_graph=True # <-- important
                           )
    #dxdxy = torch.tensor(dxdxy, requires_grad=True) #<-- won't do the trick. Remove
    loss = torch.sum(dxdxy)

    loss.backward()
    optimizer.step()

    if itr % 5 == 0:
        print(loss)


Instruction: How to define neural network neuron-by-neuron?

Input:
So we want to have a random brain-like neuron mess. Meaning:

We have AxB inputs and CxD outputs.
We want to have K (where K >= CxD) neurons that are connected to each other at random.
Yet so that all K neurons are connected to at least one of AxB inputs,
And all K neurons are connected to at least one CxD output.

Something like ths (here AxB=5, K=4, CxD=2):

Operations that neurons shall do are weighted summ + some reduction like LeakyReLu.
So one can imagine that when connection randomnes is controlled so that connections are localised over image patches alike layers of CNN it can produce intresting results.
How one could do such thing (handle neurons one-by-one) in PyTorch?

Output:
There are two issues here: The major issue is how to randomly sample the connections, and a minor issue of how to optimize a sparse linear layer.
As for the minor issue, you can implement a sparse fully-connected layer based on the linked answer.
As for the random connectivity, you'll have to implement it in a way that there are no "loops"


Instruction: how to find out the method definition of torch.nn.Module.parameters()

Input:
I am following this notebook:
One of the method:
def init_hidden(self, batch_size):
    ''' Initializes hidden state '''
    # Create two new tensors with sizes n_layers x batch_size x n_hidden,
    # initialized to zero, for hidden state and cell state of LSTM
    weight = next(self.parameters()).data

    if (train_on_gpu):
        hidden = (weight.new(self.n_layers, batch_size, self.n_hidden).zero_().cuda(),
              weight.new(self.n_layers, batch_size, self.n_hidden).zero_().cuda())
    else:
        hidden = (weight.new(self.n_layers, batch_size, self.n_hidden).zero_(),
                  weight.new(self.n_layers, batch_size, self.n_hidden).zero_())

    return hidden

I would like to see what type of weight is and how to use new() method, so I was trying to find out the parameters() method as the data attribute comes from paramerters() method.
Surprisingly, I cannot find where it comes from after reading the source code of nn module in PyTorch.
How do you figure out where to see the definition of methods you saw from PyTorch?

Output:
so I was trying to find out the parameters() method as the data
attribute comes from paramerters() method.
Surprisingly, I cannot find where it comes from after reading the
source code of nn module in PyTorch.

You can see the module definition under torch/nn/modules/module.py here at line 178.
You can then easily spot the parameters() method here.

How do you guys figure out where to see the definition of methods you
saw from PyTorch?

The easiest way that I myself always use, is to use VSCode's Go to Definition or its Peek -> Peek definition feature.
I believe Pycharm has a similar functionality as well.


Instruction: PyTorch: What is numpy.linalg.multi_dot() equivalent in PyTorch

Input:
I am trying to perform matrix multiplication of multiple matrices in PyTorch and was wondering what is the equivalent of numpy.linalg.multi_dot() in PyTorch?
If there isn't one, what is the next best way (in terms of speed and memory) I can do this in PyTorch?
Code:
import numpy as np
import torch

A = np.random.rand(3, 3)
B = np.random.rand(3, 3)
C = np.random.rand(3, 3)

results = np.linalg.multi_dot(A, B, C)

A_tsr = torch.tensor(A)
B_tsr = torch.tensor(B)
C_tsr = torch.tensor(C)

# What is the PyTorch equivalent of np.linalg.multi_dot()?

Many thanks!

Output:
~~Looks like one can send tensors into multi_dot~~
Looks like the numpy implementation casts everything into numpy arrays. If your tensors are on the cpu and detached this should work. Otherwise, the conversion to numpy would fail.
So in general - likely there isn't an alternative. I think your best shot is to take the multi_dot implementation, e.g. from here for numpy v1.19.0 and adjust it to handle tensors / skip the cast to numpy. Given the similar interface and the code simplicity I think that this should be pretty straightforward.


Instruction: How to replicate PyTorch's nn.functional.unfold function in Tensorflow?

Input:
I want to use tensorflow to rewrite the pytorch's torch.nn.functional.unfold function:
#input x:[16, 1, 50, 36]
x = torch.nn.functional.unfold(x, kernel_size=(5, 36), stride=3)
#output x:[16, 180, 16]

I tried to use the function tf.extract_image_patches():
x = tf.extract_image_patches(x,ksizes=[1, 1,5, 98],strides=[1, 1, 3, 1], rates=[1, 1, 1, 1],padding='VALID')
The input x.shape:[16,1,64,98]
I get the output x.shape:[16,1,20,490]
Then I reshape the X to [16,490,20], that was I expect.
But I get the error when I feed the data:
UnimplementedError (see above for traceback): Only support ksizes across space.
[[Node:hcn/ExtractImagePatches = ExtractImagePatches[T=DT_FLOAT, ksizes=[1, 1, 5, 98], padding="VALID", rates=[1, 1, 1, 1], strides=[1, 1, 3, 1], _device="/job:localhost/replica:0/task:0/device:GPU:0"](hcn/Reshape)]]

How could I use tensorflow to rewrite pytorch torch.nn.functional.unfold function to change the X?

Output:
x = tf.reshape(x, [16, 50, 36, 1])
x = tf.extract_image_patches(x, ksizes=[1, 4, 98, 1], strides=[1, 4, 1, 1], rates=[1, 1, 1, 1], padding='VALID')


Instruction: Can you accelerate torch DL training on anything other than "cuda" like "hip" or "OpenCL"?

Input:
I've noticed that torch.device can accept a range of arguments, precisely cpu, cuda, mkldnn, opengl, opencl, ideep, hip, msnpu.
However, when training deep learning models, I've only ever seen cuda or cpu being used. Very often the code looks something like this
if torch.cuda.is_available():
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

I've never seen any of the others being used, and was wondering if they can be used and how. The latest MacBooks with an AMD graphic card I believe should be able to use "hip", but is that true? And will the training speed be similar to that of using one CUDA GPU? If not, what is the point in torch.device accepting so many options if they cannot actually be used?

Output:
If you want to use a GPU for deep learning there is selection between CUDA and CUDA...
More broad answer, yes there is AMD's hip and some OpenCL implementation:

The is hip by AMD - CUDA like interface with ports of pytorch, hipCaffe, tensorflow, but

AMD's hip/rocm is supported only on Linux - no Windows or Mac OS support by rocm provided
Even if you want to use Linux with AMD GPU +  ROCM, you have to stick to GCN desrete devices (i.e. cards like rx 580, Vega 56/64 or Radeon VII), there is no hip/rocm support for RDNA devices (a year since a release) and it does not look to be any time soon, APUs aren't supported as well by hip.

Only one popular frameworks that supports OpenCL are Caffe and Keras+PlaidML. But

Caffe's issues:

Caffe seems have not being actively developed any more and somewhat outdated by todays standard
Performance of Caffe OpenCL implementation is about 1/2 of what is provided by nVidia's cuDNN and AMD's MIOpen, but it works quite OK and I used it in many cases.
Latest version had even grater performance hit https://github.com/BVLC/caffe/issues/6585 but at least you can run a version that works several changes behind
Also Caffe/OpenCL works there are still some bugs I fixed manually for OpenCL over AMD. https://github.com/BVLC/caffe/issues/6239

Keras/Plaid-ML

Keras on its own is much weaker framework in terms of ability to access lower level functionality
PlaidML performance is still 1/2 - to 1/3 of optimized NVidia's cuDNN & AMD's MIOpen-ROCM - and slower that caffe OpenCL in the tests I did
The future of non-TF backends for keras is not clear since 2.4 it requires TF...

Bottom line:

If you have GCN discrete AMD GPU and you run Linux you can use ROCM+Hip. Yet it isn't as stable as CUDA
You can try OpenCL Caffe or Keras-PlaidML - it maybe slower and mot as optimal as other solutions but have higher chances of making it work.

Edit 2021-09-14: there is a new project dlprimitives:
https://github.com/artyom-beilis/dlprimitives
that has better performance than both Caffe-OpenCL and Keras - it is ~75% performance for training in comparison to Keras/TF2, however it is under early development and has at this point much more limited set of layers that Caffe/Keras-PlaidML
The connection to pytorch is work in progress with some initial results: https://github.com/artyom-beilis/pytorch_dlprim
Disclaimer: I'm the author of this project


Instruction: How to invert a PyTorch Embedding?

Input:
I have an multi-task encoder/decoder model in PyTorch with a (trainable) torch.nn.Embedding embedding layer at the input.
In one particular task, I'd like to pre-train the model self-supervised (to re-construct masked input data) and use it for inference (to fill in gaps in data).
I guess for training time I can just measure loss as the distance between the input embedding and the output embedding... But for inference, how do I invert an Embedding to reconstruct the proper category/token the output corresponds to? I can't see e.g. a "nearest" function on the Embedding class...

Output:
You can do it quite easily:
import torch

embeddings = torch.nn.Embedding(1000, 100)
my_sample = torch.randn(1, 100)
distance = torch.norm(embeddings.weight.data - my_sample, dim=1)
nearest = torch.argmin(distance)

Assuming you have 1000 tokens with 100 dimensionality this would return nearest embedding based on euclidean distance. You could also use other metrics in similar manner.


Instruction: How to multiply torch[N,1] and torch[1,N]?

Input:
I'd like to calculate a matrix (shape[N,N]) by multiplying 2 torch vectors A(shape[N,1]) and B=A'(shape[1,N]).
When I use torch.matmul or torch.mm, I got errors or A'A(shape[1,1]).
If A is denoted as
A = [a_1, a_2, ..., a_N]', I want to calculate a matrix C whose (i,j)element is
for i in range(N):
     for j in range(N):
       C(i,j) = a_i * a_j

I'd like to calculate this quickly. Do you have any ideas?
thank you for your help!

Output:
If I understood you correctly you can do something like this :
A = torch.randint(0,5,(5,))
C = (A.view(1, -1) * A.view(-1, 1)).to(torch.float)

it produces :
tensor([[ 1.,  4.,  3.,  3.,  0.],
        [ 4., 16., 12., 12.,  0.],
        [ 3., 12.,  9.,  9.,  0.],
        [ 3., 12.,  9.,  9.,  0.],
        [ 0.,  0.,  0.,  0.,  0.]])

which is equivalent to writting :
 D = torch.zeros((5,5))
for i in range(5):
  for j in range(5):
    D[i][j] = A[i] * A[j]

which results in :
tensor([[ 1.,  4.,  3.,  3.,  0.],
        [ 4., 16., 12., 12.,  0.],
        [ 3., 12.,  9.,  9.,  0.],
        [ 3., 12.,  9.,  9.,  0.],
        [ 0.,  0.,  0.,  0.,  0.]])


Instruction: Retrieving original data from PyTorch nn.Embedding

Input:
I'm passing a dataframe with 5 categories (ex. car, bus, ...) into nn.Embedding.
When I do embedding.parameters(), I can see that there are 5tensors but how do I know which index corresponds to the original input (ex. car, bus, ...)?

Output:
You can't as tensors are unnamed (only dimensions can be named, see PyTorch's Named Tensors).
You have to keep the names in separate data container, for example (4 categories here):
import pandas as pd
import torch

df = pd.DataFrame(
    {
        "bus": [1.0, 2, 3, 4, 5],
        "car": [6.0, 7, 8, 9, 10],
        "bike": [11.0, 12, 13, 14, 15],
        "train": [16.0, 17, 18, 19, 20],
    }
)

df_data = df.to_numpy().T
df_names = list(df)

embedding = torch.nn.Embedding(df_data.shape[0], df_data.shape[1])
embedding.weight.data = torch.from_numpy(df_data)

Now you can simply use it with any index you want:
index = 1
embedding(torch.tensor(index)), df_names[index]

This would give you (tensor[6, 7, 8, 9, 10], "car") so the data and respective column name.


Instruction: PyTorch get_device_capability() Output Explanation

Input:
when running torch.cuda.get_device_capability() on my GTX 1070 I get the following output: (6, 1).
Could someone explain what this means?

Output:
As stated in the comment, those are actully your GPU's compute capability version/indicator.
To put it simply, it describes the features supported by your GPU. You can also view your GPU compute capability using GPU-Z if you are on windows.
For more information concerning the exact feature set differences you can have a look Here.

As you can see here, based on, these allows the developers to know which feature-set is available to them and thus enable some features for some hardwares if they support it and otherwise fall back to other implementations of them.
This may be useful as well.


Instruction: How can I see summary statistics (e.g. number of samples; type of data) of HuggingFace datasets?

Input:
I'm looking for suitable datasets to test a few new machine learning ideas. Is there any way to see summary statistics (e.g. number of samples; type of data) of HuggingFace datasets?
They provide descriptions here https://huggingface.co/datasets , but it's a bit hard to filter them.

Output:
Not sure if I have missed the obvious, but I think you have to code it by yourself. When you use list_datasets, you only get general information for each dataset:
from datasets import list_datasets
list_datasets(with_details=True)[1].__dict__

Output:
{'id': 'ag_news',
 'key': 'datasets/datasets/ag_news/ag_news.py',
 'lastModified': '2020-09-15T08:26:31.000Z',
 'description': "AG is a collection of more than 1 million news articles. News articles have been\ngathered from more than 2000 news sources by ComeToMyHead in more than 1 year of\nactivity. ComeToMyHead is an academic news search engine which has been running\nsince July, 2004. The dataset is provided by the academic comunity for research\npurposes in data mining (clustering, classification, etc), information retrieval\n(ranking, search, etc), xml, data compression, data streaming, and any other\nnon-commercial activity. For more information, please refer to the link\nhttp://www.di.unipi.it/~gulli/AG_corpus_of_news_articles.html .\n\nThe AG's news topic classification dataset is constructed by Xiang Zhang\n(xiang.zhang@nyu.edu) from the dataset above. It is used as a text\nclassification benchmark in the following paper: Xiang Zhang, Junbo Zhao, Yann\nLeCun. Character-level Convolutional Networks for Text Classification. Advances\nin Neural Information Processing Systems 28 (NIPS 2015).",
 'citation': '@inproceedings{Zhang2015CharacterlevelCN,\n  title={Character-level Convolutional Networks for Text Classification},\n  author={Xiang Zhang and Junbo Jake Zhao and Yann LeCun},\n  booktitle={NIPS},\n  year={2015}\n}',
 'size': 3991,
 'etag': '"560ac59ac8cb6f76ac4180562a7f9342"',
 'siblings': [datasets.S3Object('ag_news.py'),
  datasets.S3Object('dataset_infos.json'),
  datasets.S3Object('dummy/0.0.0/dummy_data.zip')],
 'author': None,
 'numModels': 1}

What you are actually looking for is the information that is provided by load_dataset:
from datasets import load_dataset
squad = load_dataset('squad')
squad

Output:
 DatasetDict({'train': Dataset(features: {'id': Value(dtype='string', id=None), 'title': Value(dtype='string', id=None), 'context': Value(dtype='string', id=None), 'question': Value(dtype='string', id=None), 'answers': Sequence(feature={'text': Value(dtype='string', id=None), 'answer_start': Value(dtype='int32', id=None)}, length=-1, id=None)}, num_rows: 87599), 'validation': Dataset(features: {'id': Value(dtype='string', id=None), 'title': Value(dtype='string', id=None), 'context': Value(dtype='string', id=None), 'question': Value(dtype='string', id=None), 'answers': Sequence(feature={'text': Value(dtype='string', id=None), 'answer_start': Value(dtype='int32', id=None)}, length=-1, id=None)}, num_rows: 10570)})

Here you get the number of samples for each split (num_rows) and the datatype of each feature. But load_dataset will load the whole dataset which can be an undesired behavior and therefore should be rejected for performance reasons.
An alternative is the following as far as I have not overlooked a parameter that allows to only load the dataset_infos.json of each dataset:
import datasets
import requests
from datasets import list_datasets
from datasets.utils.file_utils import REPO_DATASETS_URL

sets = list_datasets()
version = datasets.__version__
name = 'dataset_infos.json'
summary =[]

for d in sets:
     print('loading {}'.format(d))
     try:
         r = requests.get(REPO_DATASETS_URL.format(version=version, path=d, name=name))
         summary.append(r.json())
     except:
         print('Could not load {}'.format(d))

#the features and splits values are probably interesting for you
print(summary[0]['default']['features'])
print(summary[0]['default']['splits'])

Output:
{'email_body': {'dtype': 'string', 'id': None, '_type': 'Value'}, 'subject_line': {'dtype': 'string', 'id': None, '_type': 'Value'}}
{'test': {'name': 'test', 'num_bytes': 1384177, 'num_examples': 1906, 'dataset_name': 'aeslc'}, 'train': {'name': 'train', 'num_bytes': 11902668, 'num_examples': 14436, 'dataset_name': 'aeslc'}, 'validation': {'name': 'validation', 'num_bytes': 1660730, 'num_examples': 1960, 'dataset_name': 'aeslc'}}

P.S.: I haven't check the dataset_infos.json of the datasets that weren't loaded. They have probably a more complex structure or errors inside.