class_docstr: A3C: Asynchronous Advantage Actor-Critic.
  
  See http://arxiv.org/abs/1602.01783
  
  Args:
      model (A3CModel): Model to train
      optimizer (chainer.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      beta (float): Weight coefficient for the entropy regularizaiton term.
      process_idx (int): Index of the process.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      v_loss_coef (float): Weight coefficient for the loss of the value
          function
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      batch_states (callable): method which makes a batch of observations.
          default is `chainerrl.misc.batch_states.batch_states`
class_docstr_tok: ["A3C",":","Asynchronous","Advantage","Actor-Critic",".","See","http",":","\/\/arxiv.org\/abs\/1602.01783","Args",":","model","(","A3CModel",")",":","Model","to","train","optimizer","(","chainer.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","beta","(","float",")",":","Weight","coefficient","for","the","entropy","regularizaiton","term",".","process_idx","(","int",")",":","Index","of","the","process",".","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","v_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","batch_states","(","callable",")",":","method","which","makes","a","batch","of","observations",".","default","is","`","chainerrl.misc.batch_states.batch_states","`"]
code: def update(self, statevar):
      assert self.t_start < self.t
  
      if statevar is None:
          R = 0
      else:
          with state_kept(self.model):
              _, vout = self.model.pi_and_v(statevar)
          R = float(vout.array)
  
      pi_loss = 0
      v_loss = 0
      for i in reversed(range(self.t_start, self.t)):
          R *= self.gamma
          R += self.past_rewards[i]
          if self.use_average_reward:
              R -= self.average_reward
          v = self.past_values[i]
          advantage = R - v
          if self.use_average_reward:
              self.average_reward += self.average_reward_tau * float(
                  advantage.array
              )
          # Accumulate gradients of policy
          log_prob = self.past_action_log_prob[i]
          entropy = self.past_action_entropy[i]
  
          # Log probability is increased proportionally to advantage
          pi_loss -= log_prob * float(advantage.array)
          # Entropy is maximized
          pi_loss -= self.beta * entropy
          # Accumulate gradients of value function
  
          v_loss += (v - R) ** 2 / 2
  
      if self.pi_loss_coef != 1.0:
          pi_loss *= self.pi_loss_coef
  
      if self.v_loss_coef != 1.0:
          v_loss *= self.v_loss_coef
  
      # Normalize the loss of sequences truncated by terminal states
      if (
          self.keep_loss_scale_same
          and self.t - self.t_start < self.t_max
      ):
          factor = self.t_max / (self.t - self.t_start)
          pi_loss *= factor
          v_loss *= factor
  
      if self.normalize_grad_by_t_max:
          pi_loss /= self.t - self.t_start
          v_loss /= self.t - self.t_start
  
      if self.process_idx == 0:
          logger.debug(
              "pi_loss:%s v_loss:%s", pi_loss.array, v_loss.array
          )
  
      total_loss = F.squeeze(pi_loss) + F.squeeze(v_loss)
  
      # Compute gradients using thread-specific model
      self.model.cleargrads()
      total_loss.backward()
      # Copy the gradients to the globally shared model
      copy_param.copy_grad(
          target_link=self.shared_model, source_link=self.model
      )
      # Update the globally shared model
      if self.process_idx == 0:
          norm = sum(
              np.sum(np.square(param.grad))
              for param in self.optimizer.target.params()
              if param.grad is not None
          )
          logger.debug("grad norm:%s", norm)
      self.optimizer.update()
      if self.process_idx == 0:
          logger.debug("update")
  
      self.sync_parameters()
      if isinstance(self.model, Recurrent):
          self.model.unchain_backward()
  
      self.past_action_log_prob = {}
      self.past_action_entropy = {}
      self.past_states = {}
      self.past_rewards = {}
      self.past_values = {}
  
      self.t_start = self.t
  
imports: import copy
  from logging import getLogger
  import chainer
  from chainer import functions
  import numpy
  from chainerrl import agent
  from chainerrl.misc import async_
  from chainerrl.misc.batch_states import batch_states
  from chainerrl.misc import copy_param
  from chainerrl.recurrent import Recurrent
  from chainerrl.recurrent import RecurrentChainMixin
  from chainerrl.recurrent import state_kept
label_desc: Use image node_id 4 for calling the A3C obj's underlying member method code with example usage: obj.update(statevar) without return types