Instruction: lstm pytorch RuntimeError: Expected hidden[0] size (1, 1, 256), got (1, 611, 256)

Input:
I am trying to do batch processing with nn.lstm
From the documentation https://pytorch.org/docs/master/generated/torch.nn.LSTM.html I get that h0 and c0 should be of dimension:(num_layers * num_directions, batch, hidden_size).
But when I am trying to give input tensor with batch size>1 and h0 , c0 batch size>1. It is giving me error stating: "RuntimeError: Expected hidden[0] size (1, 1, 256), got (1, 611, 256)"
Here is my code:
it contains 1 memory buffer, Actor, Critic, TD3, ENV classes and main training is in TD3 which has actor and critic objects.
Can someone please help a check what am i missing here.
import os
import time
import random
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import StandardScaler
import torch
import torch.nn as nn
import torch.nn.functional as F
from random import random as rndm
from torch.autograd import Variable
from collections import deque
import pandas_datareader.data as pdr
import datetime

os.chdir('C:\\Users\\granthjain\\Desktop\\startup_code')

torch.set_default_tensor_type('torch.DoubleTensor')

f = open('lstm_with_noise_batch.txt', 'w+')

class ReplayBuffer(object):

    def __init__(self, max_size=1e6):
        self.storage = []
        self.max_size = max_size
        self.ptr = 0

    def add(self, transition):
        if len(self.storage) == self.max_size:
            self.storage[int(self.ptr)] = transition
        else:
            self.storage.append(transition)
        self.ptr = (self.ptr + 1) % self.max_size

    def sample(self, batch_size):

        ind = np.random.randint(0, self.ptr, size=batch_size)
        ind = np.random.randint(self.ptr)
        (batch_states, batch_next_states, batch_actions, batch_rewards,
         batch_dones) = ([], [], [], [], [])
        for i in range(ind - batch_size, ind):
            (state, next_state, action, reward, done) = self.storage[i]

            if state is None:
                continue
            elif next_state is None:
                continue
            elif action is None:
                continue
            elif reward is None:
                continue
            elif done is None:
                continue

            batch_states.append(np.array(state, copy=False))
            batch_next_states.append(np.array(next_state, copy=False))
            batch_actions.append(np.array(action, copy=False))
            batch_rewards.append(np.array(reward, copy=False))
            batch_dones.append(np.array(done, copy=False))

        return (np.array(batch_states, dtype=object).astype(float),
                np.array(batch_next_states,
                dtype=object).astype(float), np.array(batch_actions,
                dtype=object).astype(float), np.array(batch_rewards,
                dtype=object).astype(float), np.array(batch_dones,
                dtype=object).astype(float))

class Actor(nn.Module):

    def __init__(
        self,
        state_dim,
        action_dim,
        max_action,
        ):
        super(Actor, self).__init__()
        self.lstm = nn.LSTM(state_dim, 256)
        self.layer_1 = nn.Linear(256, 400)
        self.layer_2 = nn.Linear(400, 300)
        self.layer_3 = nn.Linear(300, action_dim)
        self.max_action = max_action

    def forward(self, x, hx):
        (hx, cx) = hx
        (output, (hx, cx)) = self.lstm(x, (hx, cx))
        x = F.relu(self.layer_1(output))
        x = F.relu(self.layer_2(x))
        x = self.max_action * torch.tanh(self.layer_3(x))

    # print("inside forward type cx:",len(output))

        return (x, hx, cx)

class Critic(nn.Module):

    def __init__(self, state_dim, action_dim):
        super(Critic, self).__init__()

    # Defining the first Critic neural network

        self.lstm1 = nn.LSTM(state_dim + action_dim, 256)
        self.layer_1 = nn.Linear(256, 400)
        self.layer_2 = nn.Linear(400, 300)
        self.layer_3 = nn.Linear(300, 1)

    # Defining the second Critic neural network

        self.lstm2 = nn.LSTM(state_dim + action_dim, 256)
        self.layer_4 = nn.Linear(256, 400)
        self.layer_5 = nn.Linear(400, 300)
        self.layer_6 = nn.Linear(300, 1)

    def forward(
        self,
        x,
        u,
        hx,
        ):
        xu = torch.cat([x, u], 1)

    # Forward-Propagation on the first Critic Neural Network

        xu = torch.reshape(xu, (xu.shape[0], 1, 6))
        (hx1, cx1) = hx
        (hx2, cx2) = hx
        (output, (hx1, cx1)) = self.lstm1(xu, (hx1, hx2))
        x1 = F.relu(self.layer_1(output))
        x1 = F.relu(self.layer_2(x1))
        x1 = self.layer_3(x1)

    # Forward-Propagation on the second Critic Neural Network

        (output, (hx2, cx2)) = self.lstm2(xu, (hx2, cx2))
        x2 = F.relu(self.layer_4(output))
        x2 = F.relu(self.layer_5(x2))
        x2 = self.layer_6(x2)
        return (
            x1,
            x2,
            hx1,
            hx2,
            cx1,
            cx2,
            )

    def Q1(
        self,
        x,
        u,
        hx1,
        ):
        xu = torch.cat([x, u], 1)
        xu = torch.reshape(xu, (xu.shape[0], 1, 6))
        (hx1, cx1) = hx1
        (output, (hx1, cx1)) = self.lstm1(xu, (hx1, cx1))
        x1 = F.relu(self.layer_1(output))
        x1 = F.relu(self.layer_2(x1))
        x1 = self.layer_3(x1)
        return (x1, hx1, cx1)

class TD3(object):

    def __init__(
        self,
        state_dim,
        action_dim,
        max_action,
        ):
        self.actor = Actor(state_dim, action_dim, max_action).to(device)
        self.actor_target = Actor(state_dim, action_dim,
                                  max_action).to(device)
        self.actor_target.load_state_dict(self.actor.state_dict())
        self.actor_optimizer = torch.optim.Adam(self.actor.parameters())
        self.critic = Critic(state_dim, action_dim).to(device)
        self.critic_target = Critic(state_dim, action_dim).to(device)
        self.critic_target.load_state_dict(self.critic.state_dict())
        self.critic_optimizer = \
            torch.optim.Adam(self.critic.parameters())
        self.max_action = max_action

    def select_action(self, state, hx1):
        (hx, cx) = hx1
        x = self.actor(state, hx1)
        return x

    def train(
        self,
        replay_buffer,
        iterations,
        batch_size=50,
        discount=0.99,
        tau=0.005,
        policy_noise=0.2,
        noise_clip=0.5,
        policy_freq=2,
        ):

        b_state = torch.Tensor([])
        b_next_state = torch.Tensor([])
        b_done = torch.Tensor([])
        b_reward = torch.Tensor([])
        b_action = torch.Tensor([])

        for it in range(iterations):

            # print ('it: ', it, ' iterations: ', iterations)

      # Step 4: We sample a batch of transitions (s, s’, a, r) from the memory

            (batch_states, batch_next_states, batch_actions,
             batch_rewards, batch_dones) = \
                replay_buffer.sample(batch_size)

            batch_states = batch_states.astype(float)
            batch_next_states = batch_next_states.astype(float)
            batch_actions = batch_actions.astype(float)
            batch_rewards = batch_rewards.astype(float)
            batch_dones = batch_dones.astype(float)

            state = torch.from_numpy(batch_states)
            next_state = torch.from_numpy(batch_next_states)
            action = torch.from_numpy(batch_actions)
            reward = torch.from_numpy(batch_rewards)
            done = torch.from_numpy(batch_dones)

            b_size = 1
            seq_len = state.shape[0]
            batch = b_size
            input_size = state_dim

            state = torch.reshape(state, (seq_len, 1, state_dim))
            next_state = torch.reshape(next_state, (seq_len, 1,
                    state_dim))
            done = torch.reshape(done, (seq_len, 1, 1))
            reward = torch.reshape(reward, (seq_len, 1, 1))
            action = torch.reshape(action, (seq_len, 1, action_dim))

            b_state = torch.cat((b_state, state),dim=1)
            b_next_state = torch.cat((b_next_state, next_state),dim=1)
            b_done = torch.cat((b_done, done),dim=1)
            b_reward = torch.cat((b_reward, reward),dim=1)
            b_action = torch.cat((b_action, action),dim=1)

        print("dim state:",b_state.shape)
        print("dim next_state:",b_next_state.shape)
        print("dim done:",b_done.shape)
        print("dim reward:",b_reward.shape)
        print("dim action:",b_action.shape)

      # for h and c shape (num_layers * num_directions, batch, hidden_size)

        h0 = torch.zeros(1, b_state.shape[1], 256)
        c0 = torch.zeros(1, b_state.shape[1], 256)
      # Step 5: From the next state s’, the Actor target plays the next action a’

        next_action = self.actor_target(next_state, (h0, c0))
        next_action = next_action[0]

      # Step 6: We add Gaussian noise to this next action a’ and we clamp it in a range of values supported by the environment

        noise = torch.Tensor(next_action).data.normal_(0,
                policy_noise).to(device)
        noise = noise.clamp(-noise_clip, noise_clip)
        next_action = (next_action + noise).clamp(-self.max_action,
                self.max_action)

      # Step 7: The two Critic targets take each the couple (s’, a’) as input and return two Q-values Qt1(s’,a’) and Qt2(s’,a’) as outputs

        result = self.critic_target(next_state, next_action, (h0,
                c0))
        target_Q1 = result[0]
        target_Q2 = result[1]

      # Step 8: We keep the minimum of these two Q-values: min(Qt1, Qt2)

        target_Q = torch.min(target_Q1, target_Q2).double()

      # Step 9: We get the final target of the two Critic models, which is: Qt = r + γ * min(Qt1, Qt2), where γ is the discount factor

        target_Q = reward + (1 - done) * discount * target_Q

      # Step 10: The two Critic models take each the couple (s, a) as input and return two Q-values Q1(s,a) and Q2(s,a) as outputs

        action = torch.reshape(action, next_action.shape)
        result = self.critic(state, action, (h0, c0))
        current_Q1 = result[0]
        current_Q2 = result[1]

      # Step 11: We compute the loss coming from the two Critic models: Critic Loss = MSE_Loss(Q1(s,a), Qt) + MSE_Loss(Q2(s,a), Qt)

        critic_loss = F.mse_loss(current_Q1, target_Q) \
            + F.mse_loss(current_Q2, target_Q)

      # Step 12: We backpropagate this Critic loss and update the parameters of the two Critic models with a SGD optimizer

        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        self.critic_optimizer.step()

      # Step 13: Once every two iterations, we update our Actor model by performing gradient ascent on the output of the first Critic model

        out = self.actor(state, (h0, c0))
        out = out[0]
        (actor_loss, hx, cx) = self.critic.Q1(state, out, (h0,
                c0))
        actor_loss = -1 * actor_loss.mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

      # Step 14: Still once every two iterations, we update the weights of the Actor target by polyak averaging

        for (param, target_param) in zip(self.actor.parameters(),
                self.actor_target.parameters()):
            target_param.data.copy_(tau * param.data + (1 - tau)
                    * target_param.data)

      # Step 15: Still once every two iterations, we update the weights of the Critic target by polyak averaging

        for (param, target_param) in zip(self.critic.parameters(),
                self.critic_target.parameters()):
            target_param.data.copy_(tau * param.data + (1 - tau)
                    * target_param.data)

  # Making a save method to save a trained model

    def save(self, filename, directory):
        torch.save(self.actor.state_dict(), '%s/%s_actor.pth'
                   % (directory, filename))
        torch.save(self.critic.state_dict(), '%s/%s_critic.pth'
                   % (directory, filename))

  # Making a load method to load a pre-trained model

    def load(self, filename, directory):
        self.actor.load_state_dict(torch.load('%s/%s_actor.pth'
                                   % (directory, filename)))
        self.critic.load_state_dict(torch.load('%s/%s_critic.pth'
                                    % (directory, filename)))

class ENV:

    def __init__(
        self,
        state_dim,
        action_dim,
        data,
        ):
        self.state_dim = state_dim
        self.state = torch.zeros(self.state_dim)
        self.state[state_dim - 1] = 100000.0
        self.next_state = torch.zeros(self.state_dim)
        self.next_state[state_dim - 1] = 100000.0
        self.action_dim = action_dim
        self.data = data
        self.idx = 0
        self._max_episode_steps = 200
        self.state[1] = self.data[self.idx]
        self.next_state[1] = self.data[self.idx]
        self.buy = 0

    def reset(self):
        self.next_state = torch.zeros(self.state_dim)
        self.next_state[state_dim - 1] = 100000.0
        self.state = torch.zeros(self.state_dim)
        self.state[state_dim - 1] = 100000.0
        self.state[1] = self.data[self.idx]
        self.next_state[1] = self.data[self.idx]

        ch = self.state[0]
        cp = self.state[1]
        cc = self.state[2]
        st = torch.tensor([ch, cp, cc])
        self.buy = 0
        return st

    def step(self, action):
        done = False
        act_t = torch.argmax(action)
        self.idx += 1
        if act_t == 0:
            cp = 1.0003 * self.state[1]
            num_s = int(self.state[2] / cp)

            self.next_state[0] += num_s
            self.next_state[2] = self.state[2] % cp

            self.next_state[1] = self.data[self.idx]
            self.buy = 1
        elif act_t == 1:
            self.next_state[1] = self.data[self.idx]
        elif act_t == 2:
            self.next_state[2] = self.state[2] + self.state[1] * (1
                    - 0.0023) * self.state[0]
            self.next_state[0] = 0
            self.next_state[1] = self.data[self.idx]

            if self.buy == 1:
                done = True
                self.buy = 0

        reward = self.next_state[2] - self.state[2] \
            + self.next_state[1] * self.next_state[0] - self.state[1] \
            * self.state[0] - 1

        self.state[0] = self.next_state[0]
        self.state[1] = self.next_state[1]
        self.state[2] = self.next_state[2]

        ch = self.state[0]
        cp = self.state[1]
        cc = self.state[2]

        st = torch.tensor([ch, cp, cc])

        return (st, reward, done)

# Selecting the device (CPU or GPU)

device = torch.device(('cuda' if torch.cuda.is_available() else 'cpu'))

# set the parameters

start_timesteps = 1e3  # Number of iterations/timesteps before which the model randomly chooses an action, and after which it starts to use the policy network
eval_freq = 5e1  # How often the evaluation step is performed (after how many timesteps)
max_timesteps = 5e3  # Total number of iterations/timesteps
save_models = True  # Boolean checker whether or not to save the pre-trained model
expl_noise = 0.1  # Exploration noise - STD value of exploration Gaussian noise
batch_size = 200  # Size of the batch
discount = 0.99  # Discount factor gamma, used in the calculation of the total discounted reward
tau = 0.005  # Target network update rate
policy_noise = 0.2  # STD of Gaussian noise added to the actions for the exploration purposes
noise_clip = 0.5  # Maximum value of the Gaussian noise added to the actions (policy)
policy_freq = 2  # Number of iterations to wait before the policy network (Actor model) is updated

state_dim = 3
action_dim = 3
max_action = 1
idx = 0

# instantiate policy

policy = TD3(state_dim, action_dim, max_action)

indices = pd.read_csv('nifty_test.csv')
indices = indices['0']

indices = pd.read_csv('EQUITY_L.csv')
indices = indices['SYMBOL']

# Create the environment for each ticker
# data = pd.read_csv('PAGEIND.csv')

for ticker in indices:
    print(ticker)

    ohlcv = pd.read_csv(ticker + '.csv')
    data = ohlcv.copy()
    data = data['Close']
    data = np.array(data).reshape(-1, 1)
    count = 0
    max_timesteps = data.shape[0]

    data = torch.DoubleTensor(data)
    env = ENV(state_dim, action_dim, data)

    replay_buffer = ReplayBuffer()

    # init training variables

    total_timesteps = 0
    timesteps_since_eval = 0
    episode_num = 0
    done = True
    t0 = time.time()
    obs = env.reset()
    hx = torch.zeros(1, 1, 256)
    cx = torch.zeros(1, 1, 256)

    # Set rewards and episode timesteps to zero

    episode_reward = 0
    episode_timesteps = 0
    episode_num = 0

    # We start the main loop over max_timesteps

    while total_timesteps < max_timesteps:

      # If the episode is done

        if done | (total_timesteps == max_timesteps - 2) \
            & (episode_timesteps > 200):
            count = count + 1
            if (count % 100 == 0) & (count >= 100) \
                | (total_timesteps == max_timesteps - 2) \
                & (episode_timesteps > 200):

            # If we are not at the very beginning, we start the training process of the model

                if total_timesteps != 0:
                    print('Total Timesteps: {} Episode Num: {} Reward: {}'.format(total_timesteps,
                            episode_num, episode_reward))
                    policy.train(
                        replay_buffer,
                        episode_timesteps,
                        batch_size,
                        discount,
                        tau,
                        policy_noise,
                        noise_clip,
                        policy_freq,
                        )

                    if total_timesteps > 0.6 * max_timesteps + 1:
                        print('model output: Total Timesteps: {} Episode Num: {} Reward: {}'.format(total_timesteps,
                                episode_num, episode_reward))
                        f.write('model output: Total Timesteps: '
                                + str(total_timesteps)
                                + ' episode_num '
                                + str(episode_num)
                                + ' episode_reward '
                                + str(episode_reward))

            # When the training step is done, we reset the state of the environment

                obs = env.reset()

            # Set the Done to False

                done = False

            # Set rewards and episode timesteps to zero

                episode_reward = 0
                episode_timesteps = 0
                episode_num += 1
                hx = torch.zeros(1, 1, 256)
                cx = torch.zeros(1, 1, 256)

      # Before 1000 timesteps, we play random actions

        if total_timesteps < 0.6 * max_timesteps:

    # random action

            actn = torch.randn(action_dim)
            action = torch.zeros(action_dim)
            action[torch.argmax(actn)] = 1
        else:

            # After 1000 timesteps, we switch to the model

    #    input of shape (seq_len, batch, input_size)

            obs1 = torch.reshape(obs, (1, 1, state_dim))
            action = policy.select_action(obs1, (hx, cx))

            actn = action[0]
            hx = action[1]
            cx = action[2]

        # If the explore_noise parameter is not 0, we add noise to the action and we clip it

            if expl_noise != 0:
                print ('policy action:', actn)
                actn = actn + torch.randn(action_dim)
                action = torch.zeros(action_dim)
                action[torch.argmax(actn)] = 1

      # The agent performs the action in the environment, then reaches the next state and receives the reward

        (new_obs, reward, done) = env.step(action)

      # We check if the episode is done

        done_bool = (0 if episode_timesteps + 1
                     == env._max_episode_steps else float(done))

      # We increase the total reward

        episode_reward += reward

      # We store the new transition into the Experience Replay memory (ReplayBuffer)

        replay_buffer.add((obs, new_obs, action, reward, done_bool))

      # We update the state, the episode timestep, the total timesteps, and the timesteps since the evaluation of the policy

        obs = new_obs
        episode_timesteps += 1
        total_timesteps += 1
        timesteps_since_eval += 1

f.close()

and below is the output:
20MICRONS
Total Timesteps: 611 Episode Num: 0 Reward: -53044.2697380831
dim state: torch.Size([200, 611, 3])
dim next_state: torch.Size([200, 611, 3])
dim done: torch.Size([200, 611, 1])
dim reward: torch.Size([200, 611, 1])
dim action: torch.Size([200, 611, 3])
Traceback (most recent call last):

  File "C:\Users\granthjain\Desktop\try_lstm.py", line 538, in <module>
    policy_freq,

  File "C:\Users\granthjain\Desktop\try_lstm.py", line 279, in train
    next_action = self.actor_target(next_state, (h0, c0))

  File "C:\Users\granthjain\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)

  File "C:\Users\granthjain\Desktop\try_lstm.py", line 106, in forward
    (output, (hx, cx)) = self.lstm(x, (hx, cx))

  File "C:\Users\granthjain\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)

  File "C:\Users\granthjain\anaconda3\lib\site-packages\torch\nn\modules\rnn.py", line 567, in forward
    self.check_forward_args(input, hx, batch_sizes)

  File "C:\Users\granthjain\anaconda3\lib\site-packages\torch\nn\modules\rnn.py", line 523, in check_forward_args
    'Expected hidden[0] size {}, got {}')

  File "C:\Users\granthjain\anaconda3\lib\site-packages\torch\nn\modules\rnn.py", line 187, in check_hidden_size
    raise RuntimeError(msg.format(expected_hidden_size, tuple(hx.size())))

RuntimeError: Expected hidden[0] size (1, 1, 256), got (1, 611, 256)

Output:
Did you make the input dimensions as required in nn.LSTM as well? I saw that you haven't set batch_first = True, and hence the input tensor has to be in the form

(seq_len, batch, input_size)


Instruction: Is it the right way to process a PIL image in pytorch?

Input:
I want to train a deep model from .png images by using Pytorch. I'm using a pre-trained model on ImageNet so I need to normalize images before feeding them to the network, but when I look at the result of the transform I see there are some values more than 1 and some less than -1. I'm wondering shouldn't they all be in [-1, 1] range? Am I doing it in the right way?
There is my code:
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
preprocess = transforms.Compose([
transforms.ToTensor(),
normalize])

x = Image.open("path/to/the/file").convert('RGB')
x = self.transform(x) # I feed the network by x

Output:
What you are doing is correct but mean and std are not calculated based on your data, rather you've taken those values from ImageNet dataset.
There will be some images which are out of [-1, 1] range as they weren't part of the mean and std calculations in the first place and it's expected during test. Also there are images outside this range as it changes mean and standard deviation to 0 and 1 respectively, hence there are samples which are outside this range.
If you wish to fine-tune your neural network you should calculate per-channel mean and std and input those values instead (though it might not make a large difference, depending on dataset and how many images you have).


Instruction: Gradient Checkpointing returning values

Input:
I have a checkpoint callback function (i.e, custom_dec) that returns a Tensor, and a dictionary. But it seems like this function does not return dictionaries (or other data types), but only tensors. What is the workaround to this, as the module that I want to checkpoint is returning a tensor, plus a data type as dictionary:
def custom_dec(self, module):
        def custom_forward(*inputs):
            output = module(inputs[0], inputs[1],
                            encoder_attn_mask=inputs[2],
                            decoder_padding_mask=inputs[3],
                            layer_state=inputs[4],
                            causal_mask=inputs[5],
                            output_attentions=inputs[6],
                            )
            # output[2] is a python dictionary
            return output[0], output[2]

The following is the checkpoint call:
x, layer_past = \
                checkpoint.checkpoint(
                    self.custom_dec(decoder_layer),
                    x,
                    encoder_hidden_states,
                    encoder_padding_mask,
                    decoder_padding_mask,
                    layer_state,
                    decoder_causal_mask,
                    output_attentions,
                )

The error:

TypeError: CheckpointFunctionBackward.forward: expected Variable (got
dictionary) for return value 1

Output:
A similar situation was discussed here.
What you can do is to convert the dictionary into some tensor form. I faced an error where it was caused by an input list which is not accepted by torch.utils.checkpoint. My solution was to pass the tensors in the list as independent tensors and form a list out of them in custom_forward.
I don't know the form of your dictionary (e.g. if every key will always have a value), but you can come up with a Dictionary-Tensor inter-change scheme that works for your dictionary.


Instruction: Why does code A runs well whereas B got an error?

Input:
I encountered a question when I try to understand a line of code:
scores = [s[tuple(k.t())] for s, k in zip(scores, keypoints)].
Here I am going to give the code that can reproduce it:
import torch # v1.5.1

scores = [torch.rand(480, 640)]
keypoints = [torch.randint(0, 480, [978, 2])]

scores = [s[tuple(k.t())] for s, k in zip(scores, keypoints)] # label A, OK

for s, k in zip(scores, keypoints):
    print(s[tuple(k.t())]) # label B, Error

>>> IndexError: too many indices for tensor of dimension 1

I think the above two kinds of code (A and B respectively) are almost the same, but the latter threw out an error. I discussed it with my roommate but we have no idea what's going on.
Could someone help me? Thanks in advance!

Output:
You can just change the order of A and B, for you have already changed "scores" when you execute B. It will definitely works.
import torch
scores = [torch.rand(2, 3)]
keypoints = [torch.randint(0, 2, [4, 2])]

for s, k in zip(scores, keypoints):
    print(s[tuple(k.t())]) # label B, Error (now ok)

scores = [s[tuple(k.t())] for s, k in zip(scores, keypoints)] # label A, OK
print(scores)

The output is
[tensor([0.3175, 0.6934, 0.2842, 0.3175])]
[tensor([0.3175, 0.6934, 0.2842, 0.3175])]

which is exactly the same.


Instruction: How to display graphs of loss and accuracy on pytorch using matplotlib

Input:
I am new to pytorch, and i would like to know how to display graphs of loss and accuraccy And how exactly should i store these values,knowing that i'm applying a cnn model for image classification using CIFAR10.
here is my current implementation :
    def train(num_epochs,optimizer,criterion,model):
        for epoch in range(num_epochs):
            for i, (images, labels) in enumerate(trainloader):
                # origin shape: [4, 3, 32, 32] = 4, 3, 1024
                # input_layer: 3 input channels, 6 output channels, 5 kernel size
                images = images.to(device)
                labels = labels.to(device)

                # Forward pass
                outputs = model(images)
                loss = criterion(outputs, labels)

                # Backward and optimize
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                if (i+1) % 2000 == 0:
                    print (f'Epoch [{epoch+1}/{num_epochs}], Step [{i+1}/{n_total_steps}], Loss: {loss.item():.4f}')

        PATH = './cnn.pth'
        torch.save(model.state_dict(), PATH)

    def test ():
        with torch.no_grad():
            n_correct = 0
            n_samples = 0
            n_class_correct = [0 for i in range(10)]
            n_class_samples = [0 for i in range(10)]

            for images, labels in testloader:
                images = images.to(device)
                labels = labels.to(device)
                outputs = model(images)
                # max returns (value ,index)
                _, predicted = torch.max(outputs, 1)
                n_samples += labels.size(0)
                n_correct += (predicted == labels).sum().item()

                for i in range(batch_size):
                    label = labels[i]
                    pred = predicted[i]

                    if (label == pred):
                        n_class_correct[label] += 1
                    n_class_samples[label] += 1

            acc = 100.0 * n_correct / n_samples
            print(f'Accuracy of the network: {acc} %')

            for i in range(10):
                acc = 100.0 * n_class_correct[i] / n_class_samples[i]
                print(f'Accuracy of {classes[i]}: {acc} %')

            test_score = np.mean([100 * n_class_correct[i] / n_class_samples[i] for i in range(10)])
            print("the score test is : {0:.3f}%".format(test_score))
            return acc

Output:
What you need to do is: Average the loss over all the batches and then append it to a variable after every epoch and then plot it. Implementation would be something like this:
import matplotlib.pyplot as plt

def my_plot(epochs, loss):
    plt.plot(epochs, loss)

def train(num_epochs,optimizer,criterion,model):
    loss_vals=  []
    for epoch in range(num_epochs):
        epoch_loss= []
        for i, (images, labels) in enumerate(trainloader):
            # rest of the code
            loss.backward()
            epoch_loss.append(loss.item())
            # rest of the code
        # rest of the code
        loss_vals.append(sum(epoch_loss)/len(epoch_loss))
        # rest of the code

    # plotting
    my_plot(np.linspace(1, num_epochs, num_epochs).astype(int), loss_vals)

my_plot([1, 2, 3, 4, 5], [100, 90, 60, 30, 10])

You can do a similar calculation for accuracy.


Instruction: "error while loading shared libraries: libnvinfer.so.7: cannot open shared object file: No such file or directory" when running TRTorch sample

Input:
I'm trying to compile my pytorch model with TRTorch engine.
I've installed TRTorch according to this link.
When the sample code is run (with the below command from this link) the given error arise:
sudo bazel run //cpp/trtorchexec -- $(realpath /home/TRTorch/tests/modules/alexnet_scripted.jit.pt) "(1,3,227,227)"

error while loading shared libraries: libnvinfer.so.7: cannot open shared object file: No such file or directory

Also, the LD_LIBRARY_PATH is set correctly.

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/TensorRT/TensorRT-7.0.0.11/lib

More info:

TRTorch: latest version (python package and binary)
TensorRT: 7.0.0.11
Pytorch: 1.5.1
CUDA: 10.2
Python: 3.6

Output:
I asked this question in TRTorch github and fixed it using:
sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/TensorRT/TensorRT-7.0.0.11/lib bazel run //cpp/trtorchexec $(realpath tests/models/alexnet_traced.jit.pt) "(32 3 227 227)"
The issue is available here.


Instruction: Is there a power function for tensors in pytorch?

Input:
Is there an equivalent of numpy.power in pytorch?
The function basically raises each element in the first tensor to the power represented in each corresponding element in the second tensor.

Output:
You are looking for torch.pow.
As mentioned by @Szymon Maszke, you can also use ** operator:
y = torch.pow(x, y)
# the same as
y = x ** y
# and also the same as
y = x.pow(y)

where y can be a scalar or a tensor with shape broadcastable to x's shape.


Instruction: Expected device cuda:0 but got device cpu in PyTorch when I have already assigned the device to be cuda

Input:
I have a following neural network code, and I get "Expected device cuda:0 but got device cpu in PyTorch" error and I can't figure out why. I assign the device to be cuda and the print line returns cuda. I've tried assign the device as device = cuda:0 as well just in case but that had no effect. Here's the code:
def run():
    torch.multiprocessing.freeze_support()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)

    metabolites = pd.read_excel("testmetabolitedata.xlsx")
    subject_metadata = pd.read_excel("testsubj.xlsx")
    metabolitesdf = pd.DataFrame(data=metabolites)
    metabolitesdf = metabolitesdf.iloc[:, 1:9153]
    subjectsdf = pd.DataFrame(data=subject_metadata)

    n_samples, n_metabolites = metabolitesdf.shape
    print(n_samples)

    #genotypes of the target gene
    print(subjectsdf['SLCO1B1_rs4149056'])
    genotypes = subjectsdf['SLCO1B1_rs4149056']
    print(genotypes)
    # print('{} unique genotypes'.format(len(set(genotypes))))

    labels = [1 if g == 1 else 0 for g in genotypes]
    print('{} samples with genotype 1 out of {}  samples ({:.1%})'.format(sum(labels), len(labels),
                                                    sum(labels) / len(labels)))

    #Insert 0 into index 0 (first) into the list for the first row with column names
    labels.insert(0, 0)
    #log transform
    log_metabol = np.log10(metabolitesdf + 1)

    #Split data into training and validation 70% / 30%
    data = torch.utils.data.TensorDataset(torch.Tensor(np.array(log_metabol)),
                                                    torch.Tensor(labels))
    train, val = torch.utils.data.random_split(data, [int(0.7 * len(data)),
                                                                len(data) - int(0.7 * len(data))])

    print('{:.0f}/{} training/total ({:.1%}) in training set, {:.0f}/{} val/total ({:.1%}) in validation set'.format(\
        train[:][1].sum(), len(train), train[:][1].sum() / len(train),
        val[:][1].sum(), len(val), val[:][1].sum() / len(val)))

    class MultiLayerPredictor(torch.nn.Module):
        def __init__(self, input_shape, output_shape=1, hidden_dim=1024, **kwargs):
            super().__init__()
            self.fc1 = torch.nn.Linear(in_features=input_shape, out_features=hidden_dim)
            self.bn1 = torch.nn.BatchNorm1d(hidden_dim)
            self.fc2 = torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim)
            self.bn2 = torch.nn.BatchNorm1d(hidden_dim)
            self.fc3 = torch.nn.Linear(in_features=hidden_dim, out_features=output_shape)

        def forward(self, x):
            l1 = torch.relu(self.bn1(self.fc1(x)))
            l2 = torch.relu(self.bn2(self.fc2(l1)))
            return torch.sigmoid(self.fc3(l2)).reshape(-1)

    #load the training and validation sets
    print("Load training and validation data ")
    train_loader = torch.utils.data.DataLoader(train, batch_size=128,
                                                shuffle=True, num_workers=10, pin_memory=True)
    val_loader = torch.utils.data.DataLoader(val, batch_size=128,
                                                shuffle=False, num_workers=10, pin_memory=True)

    print("Loading complete, create model")
    model3 = MultiLayerPredictor(input_shape=n_metabolites).to(device)
    print("Model created! Moving to optimizer")
    optimizer3 = torch.optim.SGD(model3.parameters(), lr=1e-2)
    print("Optimizer done")
    objective3 = torch.nn.BCELoss()
    epochs = 30

    print_stats_interval = 10
    log3 = []
    print("Moving to training loop")

    for epoch in range(epochs):
        loss = n_correct = 0
        model3.train()
        for batch, target in train_loader:
            batch = batch.view(-1, n_metabolites).to(device)
            optimizer3.zero_grad()
            outputs = model3(batch) # stack trace shows the issue being either on this line
            train_loss = objective3(outputs, target) # or this line
            loss += train_loss.item()
            n_correct += (target == (outputs.reshape(-1) > 0.5).float()).sum()
            train_loss.backward()
            optimizer3.step()

        loss = loss / len(train_loader)
        acc = (n_correct.float() / len(train)).numpy()
        epoch += 1

        model3.eval();
        val_loss = val_n_correct = 0
        with torch.no_grad():
            for batch, target in val_loader:
                batch = batch.view(-1, n_metabolites).to(device)
                outputs = model3(batch)
                val_loss += objective3(outputs, target)
                val_n_correct += (target == (outputs.reshape(-1) > 0.5).float()).sum()
        val_loss = (val_loss / len(val_loader)).numpy()
        val_acc = (val_n_correct.float() / len(val)).numpy()

        if (epoch % print_stats_interval) == 0 or epoch == epochs:
            print(f'epoch={epoch:.0f}, loss={loss:.5f}, val_loss={np.round(val_loss,5):.5f}, acc={np.round(acc,5):.5f}, val_acc={np.round(val_acc,5):.5f}')
        log3.append((epoch, loss, val_loss, acc, val_acc))
    log3 = pd.DataFrame(log3, columns=['epoch', 'loss', 'val_loss', 'acc', 'val_acc'])

    plt.figure(figsize=(6, 3))
    plt.plot(log3['epoch'], log3['loss'], label='Training');
    plt.plot(log3['epoch'], log3['val_loss'], label='Validation');
    plt.xlabel('Epoch'); plt.ylabel('Loss')
    plt.legend();

    val_log_mutations = val_hcc[:][0].numpy().reshape(-1)
    val_true_labels = val_hcc[:][1].numpy() + 0

    res = model3(val_hcc[:][0])
    predictions = (res.detach().numpy().reshape(-1) > 0.5) + 0
    correct = (val_true_labels == predictions) + 0
    n_correct = correct.sum()
    print('{}/{} ({:.1%}) in the validation set'.format(n_correct, len(correct), n_correct / len(correct)))
    print('Majority classifier accuracy: {:.1%}'.format((len(correct) - val_true_labels.sum()) / len(correct)))

if __name__ == '__main__':
    run()

What's going on here? The stack trace here:
Traceback (most recent call last):
  File "//ad..fi/home/h/h/Desktop/neuralnet/neuralnet_train.py", line 142, in <module>
    run()
  File "//ad..fi/home/h/h/Desktop/neuralnet/neuralnet_train.py", line 99, in run
    train_loss = objective3(outputs, target)
  File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\modules\module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\modules\loss.py", line 516, in forward
    return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
  File "C:\Users\h\AppData\Roaming\Python\Python38\site-packages\torch\nn\functional.py", line 2378, in binary_cross_entropy
    return torch._C._nn.binary_cross_entropy(
RuntimeError: expected device cuda:0 but got device cpu
PS Microsoft.PowerShell.Core\FileSystem::\\ad..fi\home\h\h\Desktop\neuralnet>

Output:
Also move the targets to CUDA in both the training and validation for loops.
 for batch, target in train_loader:
            batch,target = batch.view(-1, n_metabolites).to(device),target.to(device)
                                  .
                                  .
                                  .

for batch, target in val_loader:
            batch,target = batch.view(-1, n_metabolites).to(device),target.to(device)``
                                  .
                                  .
                                  .


Instruction: Problem with pytorch stack function with the labels of a set of images

Input:
I am trying to do run the following code https://github.com/kamenbliznashki/generative_models/blob/master/ssvae.py#L174
. Unfortunately I am encoutering a few problems (line 315,316).
More specifically I have a list of tensor images for example:
[test_dataloader.dataset[i][0] for i in [0,1,2]]

That I want to stack
torch.stack([test_dataloader.dataset[i][0] for i in [0,1,2]], dim=0)
#tensor([[[[0., 0., 0.,  ..., 0., 0., 0.],
      [0., 0., 0.,  ..., 0., 0., 0.],
      [0., 0., 0.,  ..., 0., 0., 0.],
      ...,
      [0., 0., 0.,  ..., 0., 0., 0.],
      [0., 0., 0.,  ..., 0., 0., 0.],
      [0., 0., 0.,  ..., 0., 0., 0.]]],

Everything works perfectly fine in this case. But I also have a label with those images and the stack function is not working anymore. I guess the reason is that test_dataloader.dataset[i][1] for i in [0,1,2] returns a int for every images but pytorch doesn't seem to like it. I think there was an update in pytorch since the guy pushed his code, it used to work in december 2018.
torch.stack([test_dataloader.dataset[i][1] for i in [0,1,2]], dim=0).to(args.device)
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-63-ac227397e18c> in <module>
----> 1 torch.stack([test_dataloader.dataset[i][1] for i in [0,1,2]], dim=0).to(args.device)

TypeError: expected Tensor as element 0 in argument 0, but got int

I have tried
torch.stack(torch.FloatTensor([test_dataloader.dataset[i][1] for i in [0,1,2]]), dim=0)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-43-7c4b1dd41c0f> in <module>
----> 1 torch.stack([torch.FloatTensor(test_dataloader.dataset[i][1]) for i in [0,1,2]], dim=0)

RuntimeError: stack expects each tensor to be equal size, but got [7] at entry 0 and [2] at entry 1

I have also tried, where my end goal is to one_hot those values:
def one_hot(x, label_size):
    out = torch.zeros(len(x), label_size).to(x.device)
    out[torch.arange(len(x)), x.squeeze()] = 1
    return out

y = torch.FloatTensor([test_dataloader.dataset[i][1] for i in [0,1,2]])
# > tensor([7., 2., 1.])

y = one_hot(y, args.y_dim)
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
 <ipython-input-60-2357ef4d4795> in <module>
 ----> 1 y = one_hot(y, args.y_dim)

<ipython-input-16-cf3479f01a91> in one_hot(x, label_size)
 67 def one_hot(x, label_size):
 68     out = torch.zeros(len(x), label_size).to(x.device)
---> 69     out[torch.arange(len(x)), x.squeeze()] = 1
 70     return out

IndexError: tensors used as indices must be long, byte or bool tensors

Output:
The problem with the last attempt is easy to solve. Change from torch.FloatTensor to torch.LongTensor:
y = torch.tensor([test_dataloader.dataset[i][1] for i in [0,1,2]], dtype=torch.long)


Instruction: Bidirectional RNN Implementation pytorch

Input:
Hi I am trying to understand bidirectional RNN.
> class RNN(nn.Module):
>

>
>   def __init__(self,n_vocab,n_embed,hidden_size,output_size):
>
>     super().__init__()
>
>     self.hidden_size = hidden_size
>
>     self.embedding = nn.Embedding(n_vocab+1,n_embed) ## n_vocab is unique words in dictionary ## n_embed is hyperparameter

>     self.rnn = nn.RNN(n_embed, hidden_size, num_layers = 1, batch_first = True,bidirectional = True) #
>
>     self.fc =  nn.Linear(hidden_size,output_size)
>
>   def forward(self,x):>
>
>     x = x # input batch_size * seq_length
>
>     batch_size = x.size(0)
>
>     #print('Batch Size is',batch_size)
>
>     x = self.embedding(x) # batch-size x seq_length x embedding_dimension
>
>     x,hidden =self.rnn(x)   #batch-size x seq_length x hidden_size
>
>
>
>     return x,hidden

I am returning both hidden state and output while going through tutorials some says that I need to concatenate hidden state (torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)) and in some tutorials take output state (x[:,-1,:]) but both of results come difference.
What is the correct way of doing Bidirectional RNN.

Output: Both ways are correct, depending on different conditions. If nn.RNN is bidirectional (as it is in your case), you will need to concatenate the hidden state's outputs. In case, nn.RNN is bidirectional, it will output a hidden state of shape: (num_layers * num_directions, batch, hidden_size). In the context of neural networks, when the RNN is bidirectional, we would need to concatenate the hidden states from two sides (LTR, and RTL). That's why you need to concatenate the hidden states using: torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1) which is basically the same as x[:,-1,:]).


Instruction: How do I make custom pytorch datasets structured like the torchvision datasets?

Input:
I'm new to pytorch and I'm trying to reuse a Fashion MNIST CNN (from deeplizard) to categorize my timeseries data. I'm finding it hard to understand the structure of datasets, because following this official tutorial and this SO question as best I can, I'm getting something too simple. I think this is because I don't understand OOP very well. The dataset I've made works fine in my CNN for training but then trying to analyse the results with their code I get stuck.
So I create a dataset from two pytorch tensors called features [4050, 1, 150, 6] and targets[4050]:
train_dataset = TensorDataset(features,targets) # create your datset
train_dataloader = DataLoader(train_dataset, batch_size=50, shuffle=False) # create your dataloader
print(train_dataset.__dict__.keys()) # list the attributes

I get this printed output from inspecting the attributes

dict_keys(['tensors'])

But in the Fashion MNIST tutorial they access the data like this:
train_set = torchvision.datasets.FashionMNIST(
    root='./data'
    ,train=True
    ,download=True
    ,transform=transforms.Compose([
        transforms.ToTensor()
    ])
)

train_loader = torch.utils.data.DataLoader(train_set, batch_size=1000, shuffle=True)
print(train_set.__dict__.keys()) # list the attributes

And you get this printed output from inspecting the attributes

dict_keys(['root', 'transform', 'target_transform', 'transforms',
'train', 'data', 'targets'])

My dataset works fine for training but when I get to later analysis parts of the tutorial, they want me to access parts of the dataset and I get an error:
# Analytics
prediction_loader = torch.utils.data.DataLoader(train_dataset, batch_size=50)
train_preds = get_all_preds(network, prediction_loader)
preds_correct = train_preds.argmax(dim=1).eq(train_dataset.targets).sum().item()

print('total correct:', preds_correct)
print('accuracy:', preds_correct / len(train_set))

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-73-daa87335a92a> in <module>
      4 prediction_loader = torch.utils.data.DataLoader(train_dataset, batch_size=50)
      5 train_preds = get_all_preds(network, prediction_loader)
----> 6 preds_correct = train_preds.argmax(dim=1).eq(train_dataset.targets).sum().item()
      7
      8 print('total correct:', preds_correct)

AttributeError: 'TensorDataset' object has no attribute 'targets'

Can anyone tell me what's going on here? Is this something I need to change in how I make the datasets, or can I rewrite the analysis code somehow to access the right part of the dataset?

Output:
The equivalent of .targets for TensorDatasets would be train_dataset.tensors[1].
The implementation of TensorDataset is very simple:
class TensorDataset(Dataset[Tuple[Tensor, ...]]):
    r"""Dataset wrapping tensors.
    Each sample will be retrieved by indexing tensors along the first dimension.
    Arguments:
        *tensors (Tensor): tensors that have the same size of the first dimension.
    """
    tensors: Tuple[Tensor, ...]

    def __init__(self, *tensors: Tensor) -> None:
        assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors

    def __getitem__(self, index):
        return tuple(tensor[index] for tensor in self.tensors)

    def __len__(self):
        return self.tensors[0].size(0)


Instruction: How to translate the neural network of MLP from tensorflow to pytorch

Input:
I have built up an MLP neural network using 'Tensorflow', which is stated as follow:
model_mlp=Sequential()
model_mlp.add(Dense(units=35, input_dim=train_X.shape[1], kernel_initializer='normal', activation='relu'))
model_mlp.add(Dense(units=86, kernel_initializer='normal', activation='relu'))
model_mlp.add(Dense(units=86, kernel_initializer='normal', activation='relu'))
model_mlp.add(Dense(units=10, kernel_initializer='normal', activation='relu'))
model_mlp.add(Dense(units=1))

I want to convert the above MLP code using pytorch. How to do it? I try to do it as follows:
    class MLP(nn.Module):
    def __init__(self):
        super(MLP, self).__init__()
        self.fc1 = nn.Linear(train_X.shape[1],35)
        self.fc2 = nn.Linear(35, 86)
        self.fc3 = nn.Linear(86, 86)
        self.fc4 = nn.Linear(86, 10)
        self.fc5 = nn.Linear(10, 1)
    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        x = F.relu(self.fc4(x))
        x = self.fc5(x)
        return x
    def predict(self, x_test):
        x_test = torch.from_numpy(x_test).float()
        x_test = self.forward(x_test)
        return x_test.view(-1).data.numpy()
model = MLP()

I use the same dataset but the two codes give two different answers. Code written in Tensorflow always produce a much better results than using the code written in Pytorch. I wonder if my code in pytorch is not correct.
In case my written code in PyTorch is correct, I wonder how to explain the differences. I am looking forward to any replies.

Output:
Welcome to pytorch!
I guess the problem is with the initialization of your network. That is how I would do it:
def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.xavier_normal(m.weight)  # initialize with xaver normal (called gorot in tensorflow)
        m.bias.data.fill_(0.01) # initialize bias with a constant

class MLP(nn.Module):
    def __init__(self, input_dim):
        super(MLP, self).__init__()
        self.mlp = nn.Sequential(nn.Linear(input_dim ,35), nn.ReLU(),
                                 nn.Linear(35, 86), nn.ReLU(),
                                 nn.Linear(86, 86), nn.ReLU(),
                                 nn.Linear(86, 10), nn.ReLU(),
                                 nn.Linear(10, 1), nn.ReLU())

    def forward(self, x):
        y =self.mlp(x)
        return y

model = MLP(input_dim)
model.apply(init_weights)

optimizer = Adam(model.parameters())
loss_func = BCEWithLogistLoss()

# training loop

for data, label in dataloader:
    optimizer.zero_grad()

    pred = model(data)
    loss = loss_func(pred, lable)
    loss.backward()
    optimizer.step()

Notice that in pytorch we do not call model.forward(x), but model(x). That is because nn.Module applies hooks in .__call__() that are used in the backward pass.
You can check the documentation of weight initialization here: https://pytorch.org/docs/stable/nn.init.html


Instruction: Weighted Average of PyTorch Tensors

Input: I have two Pytorch tensors of the form [y11, y12] and [y21, y22]. How do I get the weighted mean of the two tensors?

Output:
you can add two tensors using torch.add and then get the mean of output tensor using torch.mean
assuming weight as 0.6 for tensor1 and 0.4 for tensor2
example:
tensor1 = [y11, y12] * 0.6 # multiplying with weight
tensor2 = [y21, y22] * 0.4 # multiplying with weight
pt_addition_result_ex = tensor1.add(tensor2) # addition of two tensors

torch.mean(pt_addition_result_ex) # mean of output tensors


Instruction: How to get input tensor shape of an unknown PyTorch model

Input:
I am writing a python script, which converts any deep learning models from popular frameworks (TensorFlow, Keras, PyTorch) to ONNX format. Currently I have used tf2onnx for tensorflow and keras2onnx for keras to ONNX conversion, and those work.
Now PyTorch has integrated ONNX support, so I can save ONNX models from PyTorch directly. But the problem is I will need input tensor shape for that model, in order to save it in ONNX format. As you already might have guessed, I am writing this script to convert unknown deep learning models.
Here is PyTorch's tutorial for ONNX conversion. There it's written:

Limitations¶
The ONNX exporter is a trace-based exporter, which means that it operates by executing your model once, and exporting the operators which were actually run during this run. This means that if your model is dynamic, e.g., changes behavior depending on input data, the export won’t be accurate.

Similarly, a trace is might be valid only for a specific input size (which is one reason why we require explicit inputs on tracing). Most of the operators export size-agnostic versions and should work on different batch sizes or input sizes. We recommend examining the model trace and making sure the traced operators look reasonable.

The code snippet I am using is this:
import torch

def convert_pytorch2onnx(self):
    """pytorch -> onnx"""

    model = torch.load(self._model_file_path)

    # Don't know how to get this INPUT_SHAPE
    dummy_input = torch.randn(INPUT_SHAPE)
    torch.onnx.export(model, dummy_input, self._onnx_file_path)
    return

So how do I know the INPUT_SHAPE of the input tensor of that unknown PyTorch model? Or is there any other way to convert the PyTorch model to ONNX?

Output:
you can follow this as a starting point to debug
list(model.parameters())[0].shape # weights of the first layer in the format (N,C,Kernel dimensions) # 64, 3, 7 ,7

after that get the N,C and create a tensor out of that by specially putting H,W as None like this toy example
import torch
import torchvision

net = torchvision.models.resnet18(pretrained = True)

shape_of_first_layer = list(net.parameters())[0].shape #shape_of_first_layer

N,C = shape_of_first_layer[:2]

dummy_input = torch.Tensor(N,C)

dummy_input = dummy_input[...,:, None,None] #adding the None for height and weight

torch.onnx.export(net, dummy_input, './alpha')


Instruction: How to use PyTorch's torchaudio in Android?

Input:
Starting from here, I downloaded the tutorial project and got it to build and run. Then I tried adding this to the project's app build.gradle (after upping the pytorch version to 1.5.0):
implementation 'org.pytorch:pytorch_android_torchaudio:1.5.0'

And I got this error:
Could not find org.pytorch:pytorch_android_torchaudio:1.5.0.

Anyone else had any luck getting PyTorch's torchaudio to work in Android?

Output:
Got this answer from here:

There is no android package dedicated for torchaudio. You build your model or pipeline in Python, then dump it as a Torchscript file, then load it from your app and run it with Torchscript runtime. Please refer to the following.

https://pytorch.org/mobile/home/

https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html


Instruction: Reinforcement learning actor predicting same actions during initial training

Input:
I have a reinforcement learning Actor Critic model with lstm.
During initial training it is giving same action value for all the states.
Can someone expert in AI/RL please help to let me know if this is normal behavior during training?
Also can you please help to let me know what should be the ideal size of lstm and linear layers if I have a state_dimension = 50 and action_dimension = 3.
Thanks in advance

Output:
This can be caused by numerous things:
1 - Check weights initialization
2 - Check the interface on which the model makes the inference, and if there is no other things preventing it from make the action choice other than the activation of that specific neuron
3 - Check your reward function. Avoid too large negative rewards. Also, if takes the same action is not an obvious way to avoid negative rewards.


Instruction: ImportError: cannot import name 'AutoModelWithLMHead' from 'transformers'

Input:
This is literally all the code that I am trying to run:
from transformers import AutoModelWithLMHead, AutoTokenizer
import torch

tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
model = AutoModelWithLMHead.from_pretrained("microsoft/DialoGPT-small")

I am getting this error:
---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-14-aad2e7a08a74> in <module>
----> 1 from transformers import AutoModelWithLMHead, AutoTokenizer
      2 import torch
      3
      4 tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
      5 model = AutoModelWithLMHead.from_pretrained("microsoft/DialoGPT-small")

ImportError: cannot import name 'AutoModelWithLMHead' from 'transformers' (c:\python38\lib\site-packages\transformers\__init__.py)

What do I do about it?

Output:
I solved it! Apperantly AutoModelWithLMHead is removed on my version.
Now you need to use AutoModelForCausalLM for causal language models, AutoModelForMaskedLM for masked language models and AutoModelForSeq2SeqLM for encoder-decoder models.
So in my case code looks like this:
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
model = AutoModelForCausalLM.from_pretrained("microsoft/DialoGPT-small")


Instruction: Sagemaker Training Job Not Uploading/Saving Training Model to S3 Output Path

Input:
Ok I've been dealing with this issue in Sagemaker for almost a week and I'm ready to pull my hair out. I've got a custom training script paired with a data processing script in a BYO algorithm Docker deployment type scenario. It's a Pytorch model built with Python 3.x, and the BYO Docker file was originally built for Python 2, but I can't see an issue with the problem that I am having.....which is that after a successful training run Sagemaker doesn't save the model to the target S3 bucket.
I've searched far and wide and can't seem to find an applicable answer anywhere. This is all done inside a Notebook instance. Note: I am using this as a contractor and don't have full permissions to the rest of AWS, including downloading the Docker image.
Dockerfile:
FROM ubuntu:18.04

MAINTAINER Amazon AI <sage-learner@amazon.com>

RUN apt-get -y update && apt-get install -y --no-install-recommends \
         wget \
         python-pip \
         python3-pip3
         nginx \
         ca-certificates \
    && rm -rf /var/lib/apt/lists/*

RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && \
    pip3 install future numpy torch scipy scikit-learn pandas flask gevent gunicorn && \
        rm -rf /root/.cache

ENV PYTHONUNBUFFERED=TRUE
ENV PYTHONDONTWRITEBYTECODE=TRUE
ENV PATH="/opt/program:${PATH}"

COPY decision_trees /opt/program
WORKDIR /opt/program

Docker Image Build:
%%sh

algorithm_name="name-this-algo"

cd container

chmod +x decision_trees/train
chmod +x decision_trees/serve

account=$(aws sts get-caller-identity --query Account --output text)

region=$(aws configure get region)
region=${region:-us-east-2}

fullname="${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest"

aws ecr describe-repositories --repository-names "${algorithm_name}" > /dev/null 2>&1

if [ $? -ne 0 ]
then
    aws ecr create-repository --repository-name "${algorithm_name}" > /dev/null
fi

# Get the login command from ECR and execute it directly
$(aws ecr get-login --region ${region} --no-include-email)

# Build the docker image locally with the image name and then push it to ECR
# with the full name.

docker build  -t ${algorithm_name} .
docker tag ${algorithm_name} ${fullname}

docker push ${fullname}

Env setup and session start:
common_prefix = "pytorch-lstm"
training_input_prefix = common_prefix + "/training-input-data"
batch_inference_input_prefix = common_prefix + "/batch-inference-input-data"

import os
from sagemaker import get_execution_role
import sagemaker as sage

sess = sage.Session()

role = get_execution_role()
print(role)

Training Directory, Image, and Estimator Setup, then a fit call:
TRAINING_WORKDIR = "a/local/directory"

training_input = sess.upload_data(TRAINING_WORKDIR, key_prefix=training_input_prefix)
print ("Training Data Location " + training_input)

account = sess.boto_session.client('sts').get_caller_identity()['Account']
region = sess.boto_session.region_name
image = '{}.dkr.ecr.{}.amazonaws.com/image-that-works:working'.format(account, region)

tree = sage.estimator.Estimator(image,
                       role, 1, 'ml.p2.xlarge',
                       output_path="s3://sagemaker-directory-that-definitely/exists",
                       sagemaker_session=sess)

tree.fit(training_input)

The above script is working, for sure. I have print statements in my script and they are printing the expected results to the console. This runs as it's supposed to, finishes up, and says that it's deploying model artifacts when IT DEFINITELY DOES NOT.
Model Deployment:
model = tree.create_model()
predictor = tree.deploy(1, 'ml.m4.xlarge')

This throws an error that the model can't be found. A call to aws sagemaker describe-training-job shows that the training was completed but I found that the time it took to upload the model was super fast, so obviously there's an error somewhere and it's not telling me. Thankfully it's not just uploading it to the aether.
{
            "Status": "Uploading",
            "StartTime": 1595982984.068,
            "EndTime": 1595982989.994,
            "StatusMessage": "Uploading generated training model"
        },

Here's what I've tried so far:

I've tried uploading it to a different bucket. I figured my permissions were the problem so I pointed it to one that I new allowed me to upload as I had done it before to that bucket. No dice.
I tried backporting the script to Python 2.x, but that caused more problems than it probably would have solved, and I don't really see how that would be the problem anyways.
I made sure the Notebook's IAM role has sufficient permissions, and it does have a SagemakerFullAccess policy

What bothers me is that there's no error log I can see. If I could be directed to that I would be happy too, but if there's some hidden Sagemaker kungfu that I don't know about I would be forever grateful.

EDIT
The training job runs and prints to both the Jupyter cell and CloudWatch as expected. I've since lost the cell output in the notebook but below is the last few lines in CloudWatch. The first number is the epoch and the rest are various custom model metrics.

Output:
Can you verify from the training job logs that your training script is running? It doesn't look like your Docker image would respond to the command train, which is what SageMaker requires, and so I suspect that your model isn't actually getting trained/saved to /opt/ml/model.
AWS documentation about how SageMaker runs the Docker container: https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-training-algo-dockerfile.html
edit: summarizing from the comments below - the training script must also save the model to /opt/ml/model (the model isn't saved automatically).


Instruction: AttributeError: 'Sequential' object has no attribute 'size'

Input:
I'm new in pytorch and just try to write a network. The data.shape is (204,6170) and the last 5 cols are some labels. The number in data is float number like 0.030822.
#%%
from sklearn.feature_selection import RFE
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from sklearn.model_selection import train_test_split
import torch.functional as F

#%%
data = pd.read_table("table.log")
data_x = data.iloc[:, 0:(data.shape[1]-5)]
data_y = data.loc[:, 'target']
X_train, X_test, y_train, y_test = train_test_split(data_x,data_y,test_size=0.2,random_state=0)

#%%
from sklearn.linear_model import LinearRegression
lr = LinearRegression(normalize=True)
lr.fit(X_train,y_train)
rfe1 = RFE(estimator=lr,n_features_to_select=2000)
rfe1 = rfe1.fit(X_train,y_train)

#%%
x_train_rfe1 = X_train[X_train.columns[rfe1.support_]]
print(x_train_rfe1.head())
class testmodel(nn.Module):
    def __init__(self):
        super(testmodel,self).__init__()
        self.conv = nn.Sequential(
            nn.Conv1d(1500, 500, 1500, 0, 0),
            nn.ReLU(),
            nn.Conv1d(500, 100, 500, 0),
            nn.ReLU(),
            nn.Conv1d(100, 20, 100, 0),
            nn.Sigmoid()
        )
    def forward(self,x):
        x = self.conv
        return x
#%%
x_train_rfe1 = torch.Tensor(x_train_rfe1.values)
y_train = torch.Tensor(y_train.values.astype(np.int64))
model = testmodel()
y = model(x_train_rfe1)

criterion = nn.MSELoss()
loss = criterion(y, y_train)
print(loss)

error message:
Traceback (most recent call last):
  File "<input>", line 7, in <module>
  File "/miniconda3/envs/ml/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/miniconda3/envs/ml/lib/python3.8/site-packages/torch/nn/modules/loss.py", line 431, in forward
    return F.mse_loss(input, target, reduction=self.reduction)
  File "/miniconda3/envs/ml/lib/python3.8/site-packages/torch/nn/functional.py", line 2203, in mse_loss
    if not (target.size() == input.size()):
  File "/miniconda3/envs/ml/lib/python3.8/site-packages/torch/nn/modules/module.py", line 575, in __getattr__
    raise AttributeError("'{}' object has no attribute '{}'".format(
AttributeError: 'Sequential' object has no attribute 'size'

Where can the error was?
Is the internet usually written like this?
How could I improve it?

Output:
You never run your input tensor x through your conv sequential layer in forward.
def forward(self, x):
    x = self.conv(x)
    return x

Do some PyTorch tutorials they will help you get the basics down: https://pytorch.org/tutorials/


Instruction: multiplying a vector (1 x N) by a tensor (N x M x M)

Input:
I am looking for a matrix operation in numpy or preferably in pytorch that allows one to multiply a vector (1 x N) by a tensor (N x M x M) and get (1 x M x M).  This is easily accomplished using a for loop, but the for loop does not allow back propagation during training. I tried using matmul in numpy and pytorch (and several others such as dot and bmm), but could not get any to work. Here is an example (where M=2, but is 256 in my use case) of what I am trying to do:
a = np.array([1,2,3])
b = np.array([[[1,2],[3,4]],[[5,6],[7,8]],[[9,10],[11,12]]])

I would like to perform the operation: 1*[[1,2],[3,4]] + 2*[[5,6],[7,8]] + 3*[[9,10],[11,12]], which can be achieved with a for loop like this:
for i in range(3):
  matrix_sum += a[i]*b[i]

Any advice or solution would be greatly appreciated.

Output:
Numpy and pytorch were built uppon matrix multiplications!
Torch example:
A = torch.rand(1, N)
B = torch.rand(N, M, M)

C = A @ B.transpose(0, 1)
C.transpose_(0, 1)
C.shape
torch.size(1, M, M)

And similarly for numpy:
A = np.random.randn(1, N)
B = np.random.randn(N, M, M)

C = A @ B.transpose(1, 0, 2)
C = C.transpose(1, 0, 2)
C.shape
(1, M, M)

Edit For the Einsum lovers:
Pytorch and numpy handle einsum pretty much in the same way:
torch.einsum('i,ijk->jk', A, B)
np.einsum('i,ijk->jk', A, B)

Pytorch einsum documentation: https://pytorch.org/docs/master/generated/torch.einsum.html
Numpy einsum documentation: https://numpy.org/doc/stable/reference/generated/numpy.einsum.html


Instruction: PyTorch 1.5.0 CUDA 10.2 installation via pip always installs CUDA 9.2

Input:
I am currently installing an environment and need pytorch 1.5.0 with CUDA 10.2 . CUDA drivers are set up and all is fine, but the pytorch download via pip is broken.
The official doc for the previous versions says the installation should go as folllows:

CUDA 10.2
pip install torch==1.5.0 torchvision==0.6.0 -f https://download.pytorch.org/whl/torch_stable.html
CUDA 10.1
pip install torch==1.5.0+cu101 torchvision==0.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html
CUDA 9.2
pip install torch==1.5.0+cu92 torchvision==0.6.0+cu92 -f https://download.pytorch.org/whl/torch_stable.html
CPU only
pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html

(source: https://pytorch.org/get-started/previous-versions/)
but when i try to install the first one for CUDA 10.2 it installs the one for CUDA 9.2:
$ pip install torch==1.5.0 torchvision==0.6.0 -f https://download.pytorch.org/whl/torch_stable.html
 Looking in links: https://download.pytorch.org/whl/torch_stable.html
 Collecting torch==1.5.0
 Downloading https://download.pytorch.org/whl/cu92/torch-1.5.0%2Bcu92-cp36-cp36m-linux_x86_64.whl (603.7 MB)
      |████████████████████████████████| 603.7 MB 985 bytes/s
 Collecting torchvision==0.6.0
 Downloading  https://download.pytorch.org/whl/cu92/torchvision-0.6.0%2Bcu92-cp36-cp36m-linux_x86_64.whl (6.5 MB)
      |████████████████████████████████| 6.5 MB 600 kB/s
Requirement already satisfied: future in /home/---/site-packages (from torch==1.5.0) (0.18.2)
Requirement already satisfied: numpy in /home/---/python3.6/site-packages (from torch==1.5.0) (1.19.1)
Requirement already satisfied: pillow>=4.1.1 in /home/---/site-packages (from torchvision==0.6.0) (7.2.0) Installing collected packages: torch, torchvision
Successfully installed torch-1.5.0+cu92
 torchvision-0.6.0+cu92

So, it downloads and installs the wrong version. Explicitely adding +cu102 to the version like with the other version also doesnt work, since then it gives the error:
$ pip install torch==1.5.0+cu102 torchvision==0.6.0+cu102 -f https://download.pytorch.org/whl/torch_stable.html
Looking in links: https://download.pytorch.org/whl/torch_stable.html
ERROR: Could not find a version that satisfies the requirement torch==1.5.0+cu102 (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2, 0.3.0.post4, 0.3.1, 0.4.0, 0.4.1, 1.0.0, 1.0.1, 1.0.1.post2, 1.1.0, 1.2.0, 1.2.0+cpu, 1.2.0+cu92, 1.3.0, 1.3.0+cpu, 1.3.0+cu100, 1.3.0+cu92, 1.3.1, 1.3.1+cpu, 1.3.1+cu100, 1.3.1+cu92, 1.4.0, 1.4.0+cpu, 1.4.0+cu100, 1.4.0+cu92, 1.5.0, 1.5.0+cpu, 1.5.0+cu101, 1.5.0+cu92, 1.5.1, 1.5.1+cpu, 1.5.1+cu101, 1.5.1+cu92, 1.6.0, 1.6.0+cpu, 1.6.0+cu101, 1.6.0+cu92)
ERROR: No matching distribution found for torch==1.5.0+cu102

Manually downloading the wheel and changing https://download.pytorch.org/whl/cu92/torch-1.5.0%2Bcu92-cp36-cp36m-linux_x86_64.whl by deleting the cu92 part or replacing it with cu102 doesnt work, either and results in an Error 403 from the pytorch server.
Sadly, I do rely on pip in this case and can't use the conda install command. Does anybody have a solution to this problem?

Output:
You can always manually download and install the .whl files:

PyTorch 1.5.0 (cu102, py3.6, linux)

https://download.pytorch.org/whl/cu102/torch-1.5.0-cp36-cp36m-linux_x86_64.whl

TorchVision 0.6.0 (cu102, py3.6, linux)

https://download.pytorch.org/whl/cu102/torchvision-0.6.0-cp36-cp36m-linux_x86_64.whl

You can check these links on https://download.pytorch.org/whl/torch_stable.html. Scroll down to cu102/ section. You'll find other versions there, if you need.


Instruction: Pytorch: How to load csv files in several folder

Input:
My data like:
folder 1
  part0001.csv
  part0002.csv
  ...
  part0199.csv
folder 2
  part0001.csv
  part0002.csv
  ...
  part0199.csv
folder 3
  part0001.csv
  part0002.csv
  ...
  part0199.csv

Update:
Each .csv file is about 100Mb. Both features and label are in the same .csv file. Each .csv file is like as follows.
  feat1 feat2 label
1 1     3     0
2 3     4     1
3 2     5     0
...

I want to load the samples in the .csv file in batch.

Output:
You have to build a dataset that loads them. (docs: https://pytorch.org/docs/stable/data.html#torch.utils.data.Dataset)
Example:
import torch
from torch.ults.data import Dataset
import glob2
import pandas as pd

class CustomDataset(Dataset):

    def __init__(self, root)
        self.root = root

        # make a list containing the path to all your csv files
        self.paths =  glob2.glob('src/**/*.csv)

    def __len__(self):
        return len(self.paths)

    def __getittem__(idx):

        data = pd.from_csv(self.paths[idx])
        x = data['features']
        y = data['labels']

        return x, y

That is the basic, you can modify it to sample random examples from each csv file or pre-process the data before training.
Edit
If you are just intereted in one line from the csv, there are three things you can do.

Pre-process your data and save it as one big .csv file and load it all in memory before training. That will save you from the trouble of reloading heavy files.
(If the previous is not possible because of the final file would nof fit in memory) Pre-process your data and save it  .csv file per datapoint. That will still require your dataloader to read from disc, but at least you will be loading lighter files this time.
(If preprocessing your data is not an option) Keep as much as you can in memory to avoid reloading files.

There aren't many secrets to implementing the first two solutions. The code for solution 3 should look something like this:
import torch
from torch.ults.data import Dataset
import glob2
import pandas as pd

class CustomDataset(Dataset):

    def __init__(self, root)
        self.root = root

        # make a list containing the path to all your csv files
        self.paths =  glob2.glob('src/**/*.csv)

        # dict to keep load data in memory:
        self.cache = {}

    def __len__(self):
        return len(self.paths)

    def __getittem__(idx):
        """This getittem will load data and save them in memory during training."""
        data = cache.get(idx, None)

        if data is None:

            data = pd.from_csv(self.paths[idx])

            try:
                # cache data into memory
                self.cache{idx: data}
            except OSError:

                # we may be using too much memory
                del self.cache[list(self.cache.keys())[0]]

        rnd_idx = np.random.randint(len(data))
        x = data['features'][rdn_idx]
        y = data['labels'][rdn_idx]

        return x, y


Instruction: Pytorch model.load call ambiguity

Input:
Simple one:
what does this do?
model.load_state_dict({name :
        weights_before[name] + (weights_after[name] - weights_before[name]) * outerstepsize
        for name in weights_before})

Thank you very much!

Output:
load_state_dict loads learnable parameters into neural network from dictionary.
Each layer has it's respective name and parameters. In this case you go over two dictionaries (weights_before and weights_after), weights_after are always used, but additionally the difference between parameter values are added multiplied by outerstepsize.
You can check out more in PyTorch docs.


Instruction: How to run pytorch with NVIDIA "cuda toolkit" version instead of the official conda "cudatoolkit" version?

Input:
Some questions came up from https://superuser.com/questions/1572640/do-i-need-to-install-cuda-separately-after-installing-the-nvidia-display-driver. One of these questions:
Does conda pytorch need a different version than the official non-conda / non-pip cuda toolkit at https://developer.nvidia.com/cuda-toolkit?
In other words: Can I use the NVIDIA "cuda toolkit" for a pytorch installation?
Context:
If you go through the "command helper" at https://pytorch.org/get-started/locally/, you can choose between cuda versions 9.2, 10.1, 10.2 and None.
Taking 10.2 can result in:
conda install pytorch torchvision cudatoolkit=10.2 -c pytorch

Taking "None" builds the following command, but then you also cannot use cuda in pytorch:
conda install pytorch torchvision cpuonly -c pytorch

Could I then use NVIDIA "cuda toolkit" version 10.2 as the conda cudatoolkit in order to make this command the same as if it was executed with cudatoolkit=10.2 parameter?
The question arose since pytorch installs a different version (10.2 instead of the most recent NVIDIA 11.0), and the conda install takes additional 325 MB. If both versions were 11.0 and the installation size was smaller, you might not even notice the possible difference. But now it is clear that conda carries its own cuda version which is independent from the NVIDIA one.

Output:
You can try to install PyTorch via Pip:
pip install torch torchvision

It is also official way of installing, available in "command helper" at https://pytorch.org/get-started/locally/.
It uses preinstalled CUDA and doesn't download own CUDA Toolkit.
Also you can choose the version of CUDA to install PyTorch for:
pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html


Instruction: Why is PyTorch handling similar weight update assignments differently?

Input:
The below code does work updating the weights using:
w -= lr * w.grad

But when updating the weights using
w = w - lr * w.grad

it throws me:

element 0 of tensors does not require grad and does not have a grad_fn

Why is that and shouldn't both assignments be equal?
import torch

X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)
y = torch.tensor([2, 4, 6, 8], dtype=torch.float32)
w = torch.tensor(0.0, dtype=torch.float32, requires_grad=True)
epochs = 10
lr = 0.002

for epoch in range(1, epochs + 1):
    y_pred = w * X
    loss = ((y_pred - y)**2).mean()
    loss.backward()

    print(w.grad)
    with torch.no_grad():
        ### Option 1 - doesn't work
        w = w - lr * w.grad

        ### Option 2 - does work
        w -= lr * w.grad

    w.grad.zero_()

Output:
The difference is that -= is an in-place op and the alternative is not. Therefore, when using -= inside the .no_grad() context, the variable will compute the operation, but the gradient won't take that op into account.
When you perform a normal subtraction, you'd expect a SubBackward as grad_fn:
import torch
x = torch.tensor([3.], requires_grad=True)
print(x)
# >>> tensor([3.], requires_grad=True)

x = x - 2
print(x)
# >>> tensor([1.], grad_fn=<SubBackward0>)

and indeed, this is what we get. But, if we try -= inside the .no_grad() context:
with torch.no_grad():
    x -= 2

print(x)
# >>> tensor([1.], requires_grad=True)

we get the expected result (i.e., 2), but no backward function (ofc, we specified that with .no_grad()). Note that it still requires_grad=True. However, if we try to run this in-place op out of the .no_grad() context, this is what happens:
x -= 2
# >>> Traceback (most recent call last):
# >>>   File "<stdin>", line 1, in <module>
# >>> RuntimeError: a leaf Variable that requires grad has been used in an in-place operation.

and if we try to run the normal subtraction inside the .no_grad() context, we will get:
x = x - 2
print(x)
# >>> tensor([-1.])

a tensor without requires_grad; and that is the reason you get the error when using this option.


Instruction: AttributeError: 'numpy.ndarray' object has no attribute 'dim' with pytorch and scipy.optimize minimize

Input:
I have the following small code snippet:
import torch
from scipy.optimize import minimize

def f(x):
    return torch.norm(x)

x = torch.tensor([1.0, 1.0])
y = minimize(f, x)
print(y)

However, this results in this error message:
> AttributeError                            Traceback (most recent call
> last) <ipython-input-17-cb070be7a142> in <module>
>       6
>       7 x = [1.0, 1.0]
> ----> 8 y = minimize(f, x)
>       9 print(y)
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\_minimize.py in
> minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints,
> tol, callback, options)
>     610         return _minimize_cg(fun, x0, args, jac, callback, **options)
>     611     elif meth == 'bfgs':
> --> 612         return _minimize_bfgs(fun, x0, args, jac, callback, **options)
>     613     elif meth == 'newton-cg':
>     614         return _minimize_newtoncg(fun, x0, args, jac, hess, hessp, callback,
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\optimize.py in
> _minimize_bfgs(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, finite_diff_rel_step, **unknown_options)    1100
> 1101     sf = _prepare_scalar_function(fun, x0, jac, args=args,
> epsilon=eps,
> -> 1102                                   finite_diff_rel_step=finite_diff_rel_step)    1103     1104     f =
> sf.fun
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\optimize.py in
> _prepare_scalar_function(fun, x0, jac, args, bounds, epsilon, finite_diff_rel_step, hess)
>     260     # calculation reduces overall function evaluations.
>     261     sf = ScalarFunction(fun, x0, args, grad, hess,
> --> 262                         finite_diff_rel_step, bounds, epsilon=epsilon)
>     263
>     264     return sf
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py
> in __init__(self, fun, x0, args, grad, hess, finite_diff_rel_step,
> finite_diff_bounds, epsilon)
>      74
>      75         self._update_fun_impl = update_fun
> ---> 76         self._update_fun()
>      77
>      78         # Gradient evaluation
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py
> in _update_fun(self)
>     164     def _update_fun(self):
>     165         if not self.f_updated:
> --> 166             self._update_fun_impl()
>     167             self.f_updated = True
>     168
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py
> in update_fun()
>      71
>      72         def update_fun():
> ---> 73             self.f = fun_wrapped(self.x)
>      74
>      75         self._update_fun_impl = update_fun
>
> ~\Anaconda3\lib\site-packages\scipy\optimize\_differentiable_functions.py
> in fun_wrapped(x)
>      68         def fun_wrapped(x):
>      69             self.nfev += 1
> ---> 70             return fun(x, *args)
>      71
>      72         def update_fun():
>
> <ipython-input-17-cb070be7a142> in f(x)
>       3
>       4 def f(x):
> ----> 5     return torch.norm(x)
>       6
>       7 x = [1.0, 1.0]
>
> ~\Anaconda3\lib\site-packages\torch\functional.py in norm(input, p,
> dim, keepdim, out, dtype)
>     738         (tensor(3.7417), tensor(11.2250))
>     739     """
> --> 740     ndim = input.dim()
>     741
>     742     # catch default case
>
> AttributeError: 'numpy.ndarray' object has no attribute 'dim'

I'm not sure why this is happening as I don't think I'm converting anything to a numpy.ndarray.
Any help would be appreciated.

Output:
You can use NumPy instead of torch.
import numpy
from scipy.optimize import minimize

def f(x):
    return numpy.linalg.norm(x)

x = numpy.array([1.0, 1.0])
y = minimize(f, x)
print(y)


Instruction: PyTorch training with dropout and/or batch-normalization

Input:
A model should be set in the evaluation mode for inference by calling model.eval().
Do we need to also do this during training before getting the model outputs? Like within a training epoch if the network contains one or more dropout and/or batch-normalization layers.
If this is not done then the output of the forward pass in the training epoch might be affected by the randomness in the dropout?
Many example codes do not do this and something along these lines is the common approach:
for t in range(num_epochs):
    # forward pass
    yhat = model(x)

    # get the loss
    loss = criterion(yhat , y)

    # backward pass, optimizer step
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

For example here is an example code to look at : convolutional_neural_network/main.py
Should this instead be?
for t in range(num_epochs):
    # forward pass
    model.eval() # disable dropout etc
    yhat = model(x)

    # get the loss
    loss = criterion(yhat , y)

    # backward pass, optimizer step
    model.train()
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

Output:
TLDR:

Should this instead be?

No!

Why?

More explanation:
Different Modules behave differently depending on whether they are in training or evaluation/test mode.
BatchNorm and Dropout are only two examples of such modules, basically any module that has a training phase follows this rule.
When you do .eval(), you are signaling all modules in the model to shift operations accordingly.
Update
The answer is during training you should not use eval mode and yes, as long as you have not set the eval mode, the dropout will be active and act randomly in each forward passes. Similarly all other modules that have two phases, will perform accordingly. That is BN will always update the mean/var for each pass, and also if you use batch_size of 1, it will error out as it can not do BN with batch of 1
As it was pointed out in comments, it should be noted that during training, you should not do eval() before the forward pass, as it effectively disables all modules that has different phases for train/test mode such as BN and Dropout (basically any module that has updateable/learnable parameters, or impacts network topology like dropout) will be disabled and you will not see them contributing to your network learning. So don't code like that!
Let me explain a bit what happens during training:
When you are in training mode, all of your modules that make up your model may have two modes, training and test mode. These modules either have learnable parameters that need to be updated during training, like BN, or affect network topology in a sense like Dropout (by disabling some features during forward pass). some modules such as ReLU() only operate in one mode and thus do not have any change when modes change.
When you are in training mode, you feed an image, it passes trough layers until it faces a dropout and here, some features are disabled, thus theri responses to the next layer is omitted, the output goes to other layers until it reaches the end of the network and you get a prediction.
the network may have correct or wrong predictions, which will accordingly update the weights. if the answer was right, the features/combinations of features that resulted in the correct answer will be positively affected and vice versa.
So during  training you do not need and should not disable dropout, as it affects the output and should be affecting it so that the model learns a better set of features.
I hope this makes it a bit more clear for you. if you still feel you need more, say so in the comments.


Instruction: How can I check the image sizes while CNN?

Input:
I'm trying to classify cat and dog in CNN with PyTorch.
While I made few layers and processing images, I found that final processed feature map size doesn't match with calculated size.
So I tried to check feature map size step by step in CNN process with print shape but it doesn't work.
I heard tensorflow enables check tensor size in steps but how can I do that?
What I want is :
        def __init__(self):
        super(CNN, self).__init__()
        conv1 = nn.Conv2d(1, 16, 3, 1, 1)
        conv1_1 = nn.Conv2d(16, 16, 3, 1, 1)
        pool1 = nn.MaxPool2d(2)
        conv2 = nn.Conv2d(16, 32, 3, 1, 1)
        conv2_1 = nn.Conv2d(32, 32, 3, 1, 1)
        pool2 = nn.MaxPool2d(2)
        conv3 = nn.Conv2d(32, 64, 3, 1, 1)
        conv3_1 = nn.Conv2d(64, 64, 3, 1, 1)
        conv3_2 = nn.Conv2d(64, 64, 3, 1, 1)
        pool3 = nn.MaxPool2d(2)

        self.conv_module = nn.Sequential(
            conv1,
            nn.ReLU(),
            conv1_1,
            nn.ReLU(),
            pool1,
            # check first result size
            conv2,
            nn.ReLU(),
            conv2_1,
            nn.ReLU(),
            pool2,
            # check second result size
            conv3,
            nn.ReLU(),
            conv3_1,
            nn.ReLU(),
            conv3_2,
            nn.ReLU(),
            pool3,
            # check third result size
            pool4,
            # check fourth result size
            pool5
            # check fifth result size
        )

If there's any other way to check feature size at every step, please give some advice.
Thanks in advance.

Output:
To do that you shouldn't use nn.Sequential. Just initialize your layers in __init__() and call them in the forward function. In the forward function you can print the shapes out. For example like this:
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()

        self.conv1 = nn.Conv2d(...)
        self.maxpool1 = nn.MaxPool2d()

        self.conv2 = nn.Conv2d(...)
        self.maxpool2 = nn.MaxPool2d()

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.maxpool1(x)
        print(x.size())

        x = self.conv2(x)
        x = F.relu(x)
        x = self.maxpool2(x)
        print(x.size())

Hope thats what you looking for!


Instruction: How can I use Numba for Pytorch tensors?

Input:
I am new to Numba and I need to use Numba to speed up some Pytorch functions. But I find even a very simple function does not work :(
import torch
import numba

@numba.njit()
def vec_add_odd_pos(a, b):
    res = 0.
    for pos in range(len(a)):
        if pos % 2 == 0:
            res += a[pos] + b[pos]
    return res

x = torch.tensor([3, 4, 5.])
y = torch.tensor([-2, 0, 1.])
z = vec_add_odd_pos(x, y)

But the following error appears
def vec_add_odd_pos(a, b):
res = 0.
^
This error may have been caused by the following argument(s):

argument 0: cannot determine Numba type of <class 'torch.Tensor'>
argument 1: cannot determine Numba type of <class 'torch.Tensor'>

Can anyone help me? A link with more examples would be also appreciated. Thanks.

Output:
numba supports numpy-arrays but not torch's tensors. There is however a bridge Tensor.numpy():

Returns self tensor as a NumPy ndarray. This tensor and the returned
ndarray share the same underlying storage. Changes to self tensor will
be reflected in the ndarray and vice versa.

That means you have to call jitted functions as:
...
z = vec_add_odd_pos(x.numpy(), y.numpy())

If z should be a torch.Tensor as well, torch.from_numpy is what we need:

Creates a Tensor from a numpy.ndarray.
The returned tensor and ndarray share the same memory. Modifications
to the tensor will be reflected in the ndarray and vice versa. The
returned tensor is not resizable.
...

For our code that means
...
z = torch.from_numpy(vec_add_odd_pos(x.numpy(), y.numpy()))

should be called.


Instruction: convert nn.Softmax to torch.tensor

Input:
I have a softmax function at the end of my Neural Network.
I want to have the probabilities as torch.tensor.For it I am using  torch.tensor(nn.softmax(x)) and getting error RuntimeError: Could not infer dtype of Softmax.
May I please know what I am doing wrong here or is there any other way to do it.

Output:
nn.Softmax is a class. You can use it like this:
import torch

x = torch.tensor([10., 3., 8.])
softmax = torch.nn.Softmax(dim=0)

probs = softmax(x)

or, you can use the Functional API torch.nn.functional.softmax:
import torch

x = torch.tensor([10., 3., 8.])

probs = torch.nn.functional.softmax(x, dim=0)

They are equivalent. In both cases, you can check that type(probs) is <class 'torch.Tensor'>.


Instruction: Multidimensional tensor product in PyTorch

Input: In pytorch I have to tensors of dimensions [K,L,M] and [M,L,N]. I want to perform a standard tensor convolution product of those tensors along the middle two dimensions to obtain a [K,N] tensor. I couldn't find official documentation on how to perform those operations, perhaps it should be better done in some other library and then reconverted to pytorch tensor?

Output: If by convolution you actually mean something like contraction, you're probably looking for torch.tensordot. You can specify the indices that should be contracted.


Instruction: How can I access layers in a pytorch module by index?

Input:
I am trying to write a pytorch module with multiple layers. Since I need the intermediate outputs I cannot put them all in a Sequantial as usual. On the other hand, since there are many layers, what I have in mind is to put the layers in a list and access them by index in a loop. Below describe what I am trying to achieve:
import torch
import torch.nn as nn
import torch.optim as optim

class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()

        self.layer_list = []

        self.layer_list.append(nn.Linear(2,3))
        self.layer_list.append(nn.Linear(3,4))
        self.layer_list.append(nn.Linear(4,5))

    def forward(self, x):
        res_list = [x]
        for i in range(len(self.layer_list)):
            res_list.append(self.layer_list[i](res_list[-1]))
        return res_list

model = MyModel()
x = torch.randn(4,2)
y = model(x)

print(y)

optimizer = optim.Adam(model.parameters())

The forward method works fine, but when I want to set an optimizer the program says
ValueError: optimizer got an empty parameter list

It appears that the layers in the list are not registered here. What can I do?

Output:
If you put your layers in a python list, pytorch does not register them correctly. You have to do so using ModuleList (https://pytorch.org/docs/master/generated/torch.nn.ModuleList.html).

ModuleList can be indexed like a regular Python list, but modules it contains are properly registered, and will be visible by all Module methods.

Your code should be something like:

import torch
import torch.nn as nn
import torch.optim as optim

class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()

        self.layer_list = nn.ModuleList()  # << the only changed line! <<

        self.layer_list.append(nn.Linear(2,3))
        self.layer_list.append(nn.Linear(3,4))
        self.layer_list.append(nn.Linear(4,5))

    def forward(self, x):
        res_list = [x]
        for i in range(len(self.layer_list)):
            res_list.append(self.layer_list[i](res_list[-1]))
        return res_list

By using ModuleList you make sure all layers are registered in the computational graph.
There is also a ModuleDict that you can use if you want to index your layers by name. You can check pytorch's containers here: https://pytorch.org/docs/master/nn.html#containers


Instruction: Can I make my custom pytorch modules behave differently when train() or eval() are called?

Input: According to the official documents, using train() or eval() will have effects on certain modules. However, now I wish to achieve a similar thing with my custom module, i.e. it does something when train() is turned on, and something different when eval() is turned on. How can I do this?

Output:
Yes, you can.
As you can see in the source code, eval() and train() are basically changing a flag called self.training (note that it is called recursively):
def train(self: T, mode: bool = True) -> T:
    self.training = mode
    for module in self.children():
        module.train(mode)
    return self

def eval(self: T) -> T:
    return self.train(False)

This flag is available in every nn.Module. If your custom module inherits this base class, then it is quite simple to achieve what you want:
import torch.nn as nn

class MyCustomModule(nn.Module):
    def __init__(self):
        super().__init__()
        # [...]

    def forward(self, x):
        if self.training:
            # train() -> training logic
        else:
            # eval()  -> inference logic


Instruction: error while import pytorch module. (The specified module could not be found.)

Input:
I just newly install python 3.8 via anaconda installer and install pytorch using command
conda install pytorch torchvision cpuonly -c pytorch

when i try to import torch, I got this error message.
OSError: [WinError 126] The specified module could not be found. Error loading "C:\Users\chunc\anaconda3\lib\site-packages\torch\lib\asmjit.dll" or one of its dependencies.

I can see dll files are still in the directory.
I ran Dependency Walker and it gave me this result.
I am with this problem for a day.
What should i do if i want to use PyTorch module?

Output:
I had the same problem, you should check if you installed Microsoft Visual C++ Redistributable, because if you didn't this may lead to the DLL load failure.
Here is a link to download it: https://aka.ms/vs/16/release/vc_redist.x64.exe


Instruction: How to understand "-input[range(target.shape[0]), target]"?

Input:
In the official example of PyTorch, it gives a loss function as follows.
def nll(input, target):
    return -input[range(target.shape[0]), target].mean()

loss_func = nll

How to understand the grammar of "input[range(target.shape[0]), target]" in the above function?
"input" has a torch.Size([64, 10]) and "target" has a torch.Size([64]). Why use "range" function here?

Output:
The range function is used as a shortcut to create a vector/list/generator starting from 0 up to 64. So it is shorthand for essentially [0,1,2,...64]
To make this explicit you could do the following:
def nll(input, target):
    minputlist = list(range(target.shape[0]))
    print(minputlist )
    return -input[minputlist, target].mean()


Instruction: Memory leak issue using PyTorch IterableDataset with zarr

Input:
I'm trying to build a pytorch project on an IterableDataset with zarr as storage backend.
class Data(IterableDataset):
    def __init__(self, path, start=None, end=None):
        super(Data, self).__init__()
        store = zarr.DirectoryStore(path)
        self.array = zarr.open(store, mode='r')

        if start is None:
            start = 0
        if end is None:
            end = self.array.shape[0]

        assert end > start

        self.start = start
        self.end = end

    def __iter__(self):
        return islice(self.array, self.start, self.end)

This works quite nicely with small test-datasets but once i move to my actual dataset (480 000 000 x 290) i'm running into a memory leak. I've tried logging out the python heap periodically as everything slows to a crawl, but i couldn't see anything increasing in size abnormally, so the lib i used (pympler) didn't actually catch the memory leak.
I'm kind of at my wits end, so if anybody has any idea how to further debug this, it would be greatly appreciated.
Cross-posted on PyTorch Forums.

Output:
Turns out that I had an issue in my validation routine:
with torch.no_grad():
    for batch in tqdm(testloader, **params):
        x = batch[:, 1:].to(device)
        y = batch[:, 0].unsqueeze(0).T
        y_test_pred = torch.sigmoid(sxnet(x))
        y_pred_tag = torch.round(y_test_pred)
        y_pred_list.append(y_pred_tag.cpu().numpy())
        y_list.append(y.numpy())

I originally thought that I am well clear of running into troubles with appending my results to lists, but the issue is that the result of .numpy was an array of arrays (since the original datatype was a 1xn Tensor).
Adding .flatten() on the numpy arrays has fixed this issue and the RAM consumption is now as I originally provisioned.