class_docstr: A2C: Advantage Actor-Critic.
  
  A2C is a synchronous, deterministic variant of Asynchronous Advantage
      Actor Critic (A3C).
  
  See https://arxiv.org/abs/1708.05144
  
  Args:
      model (nn.Module): Model to train
      optimizer (torch.optim.Optimizer): optimizer used to train the model
      gamma (float): Discount factor [0,1]
      num_processes (int): The number of processes
      gpu (int): GPU device id if not None nor negative.
      update_steps (int): The number of update steps
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      v_loss_coef (float): Weight coefficient for the loss of the value
          function
      entropy_coeff (float): Weight coefficient for the loss of the entropy
      use_gae (bool): use generalized advantage estimation(GAE)
      tau (float): gae parameter
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      max_grad_norm (float or None): Maximum L2 norm of the gradient used for
          gradient clipping. If set to None, the gradient is not clipped.
      average_actor_loss_decay (float): Decay rate of average actor loss.
          Used only to record statistics.
      average_entropy_decay (float): Decay rate of average entropy. Used only
          to record statistics.
      average_value_decay (float): Decay rate of average value. Used only
          to record statistics.
      batch_states (callable): method which makes a batch of observations.
          default is `pfrl.utils.batch_states.batch_states`
class_docstr_tok: ["A2C",":","Advantage","Actor-Critic",".","A2C","is","a","synchronous",",","deterministic","variant","of","Asynchronous","Advantage","Actor","Critic","(","A3C",")",".","See","https",":","\/\/arxiv.org\/abs\/1708.05144","Args",":","model","(","nn.Module",")",":","Model","to","train","optimizer","(","torch.optim.Optimizer",")",":","optimizer","used","to","train","the","model","gamma","(","float",")",":","Discount","factor","[","0,1","]","num_processes","(","int",")",":","The","number","of","processes","gpu","(","int",")",":","GPU","device","id","if","not","None","nor","negative",".","update_steps","(","int",")",":","The","number","of","update","steps","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","v_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","entropy_coeff","(","float",")",":","Weight","coefficient","for","the","loss","of","the","entropy","use_gae","(","bool",")",":","use","generalized","advantage","estimation","(","GAE",")","tau","(","float",")",":","gae","parameter","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","max_grad_norm","(","float","or","None",")",":","Maximum","L2","norm","of","the","gradient","used","for","gradient","clipping",".","If","set","to","None",",","the","gradient","is","not","clipped",".","average_actor_loss_decay","(","float",")",":","Decay","rate","of","average","actor","loss",".","Used","only","to","record","statistics",".","average_entropy_decay","(","float",")",":","Decay","rate","of","average","entropy",".","Used","only","to","record","statistics",".","average_value_decay","(","float",")",":","Decay","rate","of","average","value",".","Used","only","to","record","statistics",".","batch_states","(","callable",")",":","method","which","makes","a","batch","of","observations",".","default","is","`","pfrl.utils.batch_states.batch_states","`"]
code: def update(self):
      with torch.no_grad():
          _, next_value = self.model(self.states[-1])
          next_value = next_value[:, 0]
  
      self._compute_returns(next_value)
      pout, values = self.model(
          self.states[:-1].reshape(-1, *self.obs_shape)
      )
  
      actions = self.actions.reshape(-1, *self.action_shape)
      dist_entropy = pout.entropy().mean()
      action_log_probs = pout.log_prob(actions)
  
      values = values.reshape((self.update_steps, self.num_processes))
      action_log_probs = action_log_probs.reshape(
          (self.update_steps, self.num_processes)
      )
      advantages = self.returns[:-1] - values
      value_loss = (advantages * advantages).mean()
      action_loss = -(advantages.detach() * action_log_probs).mean()
  
      self.optimizer.zero_grad()
  
      (
          value_loss * self.v_loss_coef
          + action_loss * self.pi_loss_coef
          - dist_entropy * self.entropy_coeff
      ).backward()
  
      if self.max_grad_norm is not None:
          clip_l2_grad_norm_(
              self.model.parameters(), self.max_grad_norm
          )
      self.optimizer.step()
      self.states[0] = self.states[-1]
  
      self.t_start = self.t
  
      # Update stats
      self.average_actor_loss += (1 - self.average_actor_loss_decay) * (
          float(action_loss) - self.average_actor_loss
      )
      self.average_value += (1 - self.average_value_decay) * (
          float(value_loss) - self.average_value
      )
      self.average_entropy += (1 - self.average_entropy_decay) * (
          float(dist_entropy) - self.average_entropy
      )
  
imports: import warnings
  from logging import getLogger
  import torch
  from pfrl import agent
  from pfrl.utils import clip_l2_grad_norm_
  from pfrl.utils.batch_states import batch_states
  from pfrl.utils.mode_of_distribution import mode_of_distribution
label_desc: Use image node_id 4 for calling the A2C obj's underlying member method code with example usage: obj.update() without return types