class_docstr: ACER (Actor-Critic with Experience Replay).
  
  See http://arxiv.org/abs/1611.01224
  
  Args:
      model (ACERModel): Model to train. It must be a callable that accepts
          observations as input and return three values: action distributions
          (Distribution), Q values (ActionValue) and state values
          (chainer.Variable).
      optimizer (chainer.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      replay_buffer (EpisodicReplayBuffer): Replay buffer to use. If set
          None, this agent won't use experience replay.
      beta (float): Weight coefficient for the entropy regularizaiton term.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      Q_loss_coef (float): Weight coefficient for the loss of the value
          function
      use_trust_region (bool): If set true, use efficient TRPO.
      trust_region_alpha (float): Decay rate of the average model used for
          efficient TRPO.
      trust_region_delta (float): Threshold used for efficient TRPO.
      truncation_threshold (float or None): Threshold used to truncate larger
          importance weights. If set None, importance weights are not
          truncated.
      disable_online_update (bool): If set true, disable online on-policy
          update and rely only on experience replay.
      n_times_replay (int): Number of times experience replay is repeated per
          one time of online update.
      replay_start_size (int): Experience replay is disabled if the number of
          transitions in the replay buffer is lower than this value.
      normalize_loss_by_steps (bool): If set true, losses are normalized by
          the number of steps taken to accumulate the losses
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      use_Q_opc (bool): If set true, use Q_opc, a Q-value estimate without
          importance sampling, is used to compute advantage values for policy
          gradients. The original paper recommend to use in case of
          continuous action.
      average_entropy_decay (float): Decay rate of average entropy. Used only
          to record statistics.
      average_value_decay (float): Decay rate of average value. Used only
          to record statistics.
      average_kl_decay (float): Decay rate of kl value. Used only to record
          statistics.
class_docstr_tok: ["ACER","(","Actor-Critic","with","Experience","Replay",")",".","See","http",":","\/\/arxiv.org\/abs\/1611.01224","Args",":","model","(","ACERModel",")",":","Model","to","train",".","It","must","be","a","callable","that","accepts","observations","as","input","and","return","three","values",":","action","distributions","(","Distribution",")",",","Q","values","(","ActionValue",")","and","state","values","(","chainer.Variable",")",".","optimizer","(","chainer.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","replay_buffer","(","EpisodicReplayBuffer",")",":","Replay","buffer","to","use",".","If","set","None",",","this","agent","wo","n't","use","experience","replay",".","beta","(","float",")",":","Weight","coefficient","for","the","entropy","regularizaiton","term",".","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","Q_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","use_trust_region","(","bool",")",":","If","set","true",",","use","efficient","TRPO",".","trust_region_alpha","(","float",")",":","Decay","rate","of","the","average","model","used","for","efficient","TRPO",".","trust_region_delta","(","float",")",":","Threshold","used","for","efficient","TRPO",".","truncation_threshold","(","float","or","None",")",":","Threshold","used","to","truncate","larger","importance","weights",".","If","set","None",",","importance","weights","are","not","truncated",".","disable_online_update","(","bool",")",":","If","set","true",",","disable","online","on-policy","update","and","rely","only","on","experience","replay",".","n_times_replay","(","int",")",":","Number","of","times","experience","replay","is","repeated","per","one","time","of","online","update",".","replay_start_size","(","int",")",":","Experience","replay","is","disabled","if","the","number","of","transitions","in","the","replay","buffer","is","lower","than","this","value",".","normalize_loss_by_steps","(","bool",")",":","If","set","true",",","losses","are","normalized","by","the","number","of","steps","taken","to","accumulate","the","losses","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","use_Q_opc","(","bool",")",":","If","set","true",",","use","Q_opc",",","a","Q-value","estimate","without","importance","sampling",",","is","used","to","compute","advantage","values","for","policy","gradients",".","The","original","paper","recommend","to","use","in","case","of","continuous","action",".","average_entropy_decay","(","float",")",":","Decay","rate","of","average","entropy",".","Used","only","to","record","statistics",".","average_value_decay","(","float",")",":","Decay","rate","of","average","value",".","Used","only","to","record","statistics",".","average_kl_decay","(","float",")",":","Decay","rate","of","kl","value",".","Used","only","to","record","statistics","."]
code: def update(
      self,
      t_start,
      t_stop,
      R,
      states,
      actions,
      rewards,
      values,
      action_values,
      action_distribs,
      action_distribs_mu,
      avg_action_distribs,
  ):
      assert np.isscalar(R)
  
      total_loss = self.compute_loss(
          t_start=t_start,
          t_stop=t_stop,
          R=R,
          states=states,
          actions=actions,
          rewards=rewards,
          values=values,
          action_values=action_values,
          action_distribs=action_distribs,
          action_distribs_mu=action_distribs_mu,
          avg_action_distribs=avg_action_distribs,
      )
  
      # Compute gradients using thread-specific model
      self.model.cleargrads()
      F.squeeze(total_loss).backward()
      # Copy the gradients to the globally shared model
      copy_param.copy_grad(
          target_link=self.shared_model, source_link=self.model
      )
      # Update the globally shared model
      if self.process_idx == 0:
          norm = sum(
              np.sum(np.square(param.grad))
              for param in self.optimizer.target.params()
              if param.grad is not None
          )
          self.logger.debug("grad norm:%s", norm)
      self.optimizer.update()
  
      self.sync_parameters()
      if isinstance(self.model, Recurrent):
          self.model.unchain_backward()
  
imports: import copy
  from logging import getLogger
  import chainer
  from chainer import functions
  import numpy
  from chainerrl.action_value import SingleActionValue
  from chainerrl import agent
  from chainerrl import distribution
  from chainerrl import links
  from chainerrl.misc import async_
  from chainerrl.misc import copy_param
  from chainerrl.recurrent import Recurrent
  from chainerrl.recurrent import RecurrentChainMixin
  from chainerrl.recurrent import state_kept
  from chainerrl.recurrent import state_reset
label_desc: Use image node_id 7 for calling the ACER obj's underlying member method code with example usage: obj.update(t_start, t_stop, R, states, actions, rewards, values, action_values, action_distribs, action_distribs_mu, avg_action_distribs) without return types