title: Pytorch modified DQN algorithm error "the derivative for ' ' is not implemented"
body: <p>I'm trying to construct new observation and applying it into DQN.
  I use pytorch DQN algorithm with my observation code.
  its not finished yet, so data is not clear. </p>
  
  <p>I changed some line in whole code as follows.</p>
  
  <pre><code>import sys, math
  import random as rd
  import numpy as np
  import matplotlib
  import matplotlib.pyplot as plt
  
  ######################################
  from collections import namedtuple
  from itertools import count
  from PIL import Image
  
  import torch
  import torch.nn as nn
  import torch.optim as optim
  import torch.nn.functional as F
  from torch.autograd import Variable
  import torchvision.transforms as T
  
  
  Transition = namedtuple('Transition',
                      ('state', 'action', 'next_state', 'reward'))
  
  class ReplayMemory(object):
  
  def __init__(self, capacity):
      self.capacity = capacity
      self.memory = []
      self.position = 0
  
  def push(self, *args):
      """Saves a transition."""
      if len(self.memory) &lt; self.capacity:
          self.memory.append(None)
      self.memory[self.position] = Transition(*args)
      self.position = (self.position + 1) % self.capacity
  
  def sample(self, batch_size):
      return rd.sample(self.memory, batch_size)
  
  def __len__(self):
      return len(self.memory)
  
  
  class DQN(nn.Module):
  
  def __init__(self):
      super(DQN, self).__init__()
      self.l1 = nn.Linear(5, 16)
      self.l2 = nn.Linear(16, 12)
      self.l3 = nn.Linear(12, 20)
      self.head = nn.Linear(20, 1)
  def forward(self, x):
      x = F.relu(self.l1(x))
      x = F.relu(self.l2(x))
      x = F.relu(self.l3(x))
      return self.head(x.view(x.size(0), -1))
  
  BATCH_SIZE = 5
  GAMMA = 0.999
  EPS_START = 0.9
  EPS_END = 0.05
  EPS_DECAY = 200
  TARGET_UPDATE = 5
  policy_net = DQN()
  target_net = DQN()
  target_net.load_state_dict(policy_net.state_dict())
  target_net.eval()
  
  optimizer = optim.RMSprop(policy_net.parameters())
  memory = ReplayMemory(10000)
  
  def optimize_model():
  if len(memory) &lt; BATCH_SIZE:
      return
  transitions = memory.sample(BATCH_SIZE)
  # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for
  # detailed explanation).
  batch = Transition(*zip(*transitions))
  print("batch = ", batch, "\n")
  # Compute a mask of non-final states and concatenate the batch elements
  
  
  state_batch = Variable(torch.cat(batch.state))
  print("state_batch = ", state_batch)
  action_batch = Variable(torch.cat(batch.action))
  print("action_batch = ", action_batch)
  reward_batch = Variable(torch.cat(batch.reward), requires_grad = False)
  print("reward_batch = ", reward_batch)
  next_state_batch = Variable(torch.cat(batch.next_state), requires_grad = False)
  
  # Compute Q(s_t, a) - the model computes Q(s_t), then we select the
  # columns of actions taken
  state_action_values = policy_net(state_batch)
  print("state_action_values = ", state_action_values)
  
  # Compute V(s_{t+1}) for all next states.
  next_state_values = target_net(next_state_batch)
  print("next_state_values = ", next_state_values)
  
  # Compute the expected Q values
  expected_state_action_values = (next_state_values * GAMMA) + reward_batch
  print("expected next state values = ", expected_state_action_values)
  
  # Compute Huber loss
  loss = F.smooth_l1_loss(state_action_values, expected_state_action_values, reduce = False)
  
  # Optimize the model
  optimizer.zero_grad()
  loss.backward()
  for param in policy_net.parameters():
      param.grad.data.clamp_(-1, 1)
  optimizer.step()
  
  
  
  num_episodes = 5
  for i_episode in range(num_episodes):
      # Initialize the environment and state
      drive = AutoDrive(20, 20, 0, 16, 0) # x/y/yaw/velocity/heading
      drive._make_observation(0, -1, -1, -1, -1, -1) random other vehicle     location, parameters
  
      stand = 3
  
      # exploit 1
      e = 1. / ((i_episode // 100) + 1) # conditioon for choosing action
      optimizer.zero_grad()
  
      for t in range(stand):
          # Select and perform an action
  
  #         exploit 2
          if np.random.rand(1) &gt; e:
              action = rd.randint(1, 4)
          else:
              action = np.argmax(drive._select_action(0.5, 0.5)) + 1 #index + 1
          print("state = ", drive.state, ", action = ", action, ", yaw = ",     drive.yaw, ", mag = ", drive.mag)
      state = drive.state
      drive._step(action)
      drive._calc_reward(0.5, 0.5)
      print(drive.reward)
      if (drive.reward == -10):
          break
  
  
      # Store the transition in memory
      state1 = torch.FloatTensor(state).view(1, 5)
      state2 = torch.FloatTensor(drive.state).view(1, 5)
      action = torch.FloatTensor([float(action)]).view(1, 1)
      reward = torch.FloatTensor([drive.reward]).view(1, 1)
      memory.push(state1, action\
                  , state2, reward)
  
      # Perform one step of the optimization (on the target network)
      optimize_model()
      if done:
          episode_durations.append(t + 1)
          plot_durations()
          break
  # Update the target network
  if i_episode % TARGET_UPDATE == 0:
      target_net.load_state_dict(policy_net.state_dict())
  </code></pre>
  
  <p>the error occurs in the loss function</p>
  
  <pre><code>  File "&lt;ipython-input-190-29dcdbbf0383&gt;", line 1, in &lt;module&gt;
      runfile('C:/Users/desktop/.spyder-py3/temp.py',     wdir='C:/Users/desktop/.spyder-py3')
  
    File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
  execfile(filename, namespace)
  
    File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
  exec(compile(f.read(), filename, 'exec'), namespace)
  
    File "C:/Users/desktop/.spyder-py3/temp.py", line 441, in &lt;module&gt;
  optimize_model()
  
    File "C:/Users/desktop/.spyder-py3/temp.py", line 362, in optimize_model
  loss = F.smooth_l1_loss(state_action_values, expected_state_action_values, reduce = False)
  
  RuntimeError: the derivative for 'target' is not implemented
  </code></pre>
  
  <p>the inputs in the function are as follows.</p>
  
  <pre><code>expected next state values =  Variable containing:
    8.9615
   12.0198
   12.0488
   12.2920
   13.9062
  [torch.FloatTensor of size 5x1]
  
  state_action_values =  Variable containing:
   0.3765
   0.5196
   0.4587
   0.3765
   0.5636
  [torch.FloatTensor of size 5x1]
  </code></pre>
  
  <p>what i have to do? im really beginner of it, so helpful advice are welcome</p>