class_docstr: Asynchronous N-step Q-Learning.
  
  See http://arxiv.org/abs/1602.01783
  
  Args:
      q_function (A3CModel): Model to train
      optimizer (chainer.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      i_target (intn): The target model is updated after every i_target
          global steps
      explorer (Explorer): Explorer to use in training
      phi (callable): Feature extractor function
      average_q_decay (float): Decay rate of average Q, only used for
          recording statistics
      batch_states (callable): method which makes a batch of observations.
          default is `chainerrl.misc.batch_states.batch_states`
class_docstr_tok: ["Asynchronous","N-step","Q-Learning",".","See","http",":","\/\/arxiv.org\/abs\/1602.01783","Args",":","q_function","(","A3CModel",")",":","Model","to","train","optimizer","(","chainer.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","i_target","(","intn",")",":","The","target","model","is","updated","after","every","i_target","global","steps","explorer","(","Explorer",")",":","Explorer","to","use","in","training","phi","(","callable",")",":","Feature","extractor","function","average_q_decay","(","float",")",":","Decay","rate","of","average","Q",",","only","used","for","recording","statistics","batch_states","(","callable",")",":","method","which","makes","a","batch","of","observations",".","default","is","`","chainerrl.misc.batch_states.batch_states","`"]
code: def update(self, statevar):
      assert self.t_start < self.t
  
      # Update
      if statevar is None:
          R = 0
      else:
          with state_kept(self.target_q_function):
              R = float(self.target_q_function(statevar).max.array)
  
      loss = 0
      for i in reversed(range(self.t_start, self.t)):
          R *= self.gamma
          R += self.past_rewards[i]
          q = F.reshape(self.past_action_values[i], (1, 1))
          # Accumulate gradients of Q-function
          loss += F.sum(
              F.huber_loss(
                  q,
                  chainer.Variable(np.asarray([[R]], dtype=np.float32)),
                  delta=1.0,
              )
          )
  
      # Do we need to normalize losses by (self.t - self.t_start)?
      # Otherwise, loss scales can be different in case of self.t_max
      # and in case of termination.
  
      # I'm not sure but if we need to normalize losses...
      # loss /= self.t - self.t_start
  
      # Compute gradients using thread-specific model
      self.q_function.cleargrads()
      loss.backward()
      # Copy the gradients to the globally shared model
      copy_param.copy_grad(self.shared_q_function, self.q_function)
      # Update the globally shared model
      self.optimizer.update()
  
      self.sync_parameters()
      if isinstance(self.q_function, Recurrent):
          self.q_function.unchain_backward()
  
      self.past_action_values = {}
      self.past_states = {}
      self.past_rewards = {}
  
      self.t_start = self.t
  
imports: import copy
  from logging import getLogger
  import multiprocessing
  import chainer
  from chainer import functions
  import numpy
  from chainerrl.agent import AsyncAgent
  from chainerrl.agent import AttributeSavingMixin
  from chainerrl.misc import async_
  from chainerrl.misc.batch_states import batch_states
  from chainerrl.misc import copy_param
  from chainerrl.recurrent import Recurrent
  from chainerrl.recurrent import state_kept
label_desc: Use image node_id 4 for calling the NSQ obj's underlying member method code with example usage: obj.update(statevar) without return types