class_docstr: A3C: Asynchronous Advantage Actor-Critic.
  
  See http://arxiv.org/abs/1602.01783
  
  Args:
      model (A3CModel): Model to train
      optimizer (torch.optim.Optimizer): optimizer used to train the model
      t_max (int): The model is updated after every t_max local steps
      gamma (float): Discount factor [0,1]
      beta (float): Weight coefficient for the entropy regularizaiton term.
      process_idx (int): Index of the process.
      phi (callable): Feature extractor function
      pi_loss_coef (float): Weight coefficient for the loss of the policy
      v_loss_coef (float): Weight coefficient for the loss of the value
          function
      act_deterministically (bool): If set true, choose most probable actions
          in act method.
      max_grad_norm (float or None): Maximum L2 norm of the gradient used for
          gradient clipping. If set to None, the gradient is not clipped.
      recurrent (bool): If set to True, `model` is assumed to implement
          `pfrl.nn.StatelessRecurrent`.
      batch_states (callable): method which makes a batch of observations.
          default is `pfrl.utils.batch_states.batch_states`
class_docstr_tok: ["A3C",":","Asynchronous","Advantage","Actor-Critic",".","See","http",":","\/\/arxiv.org\/abs\/1602.01783","Args",":","model","(","A3CModel",")",":","Model","to","train","optimizer","(","torch.optim.Optimizer",")",":","optimizer","used","to","train","the","model","t_max","(","int",")",":","The","model","is","updated","after","every","t_max","local","steps","gamma","(","float",")",":","Discount","factor","[","0,1","]","beta","(","float",")",":","Weight","coefficient","for","the","entropy","regularizaiton","term",".","process_idx","(","int",")",":","Index","of","the","process",".","phi","(","callable",")",":","Feature","extractor","function","pi_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","policy","v_loss_coef","(","float",")",":","Weight","coefficient","for","the","loss","of","the","value","function","act_deterministically","(","bool",")",":","If","set","true",",","choose","most","probable","actions","in","act","method",".","max_grad_norm","(","float","or","None",")",":","Maximum","L2","norm","of","the","gradient","used","for","gradient","clipping",".","If","set","to","None",",","the","gradient","is","not","clipped",".","recurrent","(","bool",")",":","If","set","to","True",",","`","model","`","is","assumed","to","implement","`","pfrl.nn.StatelessRecurrent","`",".","batch_states","(","callable",")",":","method","which","makes","a","batch","of","observations",".","default","is","`","pfrl.utils.batch_states.batch_states","`"]
code: def update(self, statevar):
      assert self.t_start < self.t
  
      n = self.t - self.t_start
  
      self.assert_shared_memory()
  
      if statevar is None:
          R = 0
      else:
          with torch.no_grad(), pfrl.utils.evaluating(self.model):
              if self.recurrent:
                  (_, vout), _ = one_step_forward(
                      self.model, statevar, self.train_recurrent_states
                  )
              else:
                  _, vout = self.model(statevar)
          R = float(vout)
  
      pi_loss_factor = self.pi_loss_coef
      v_loss_factor = self.v_loss_coef
  
      # Normalize the loss of sequences truncated by terminal states
      if (
          self.keep_loss_scale_same
          and self.t - self.t_start < self.t_max
      ):
          factor = self.t_max / (self.t - self.t_start)
          pi_loss_factor *= factor
          v_loss_factor *= factor
  
      if self.normalize_grad_by_t_max:
          pi_loss_factor /= self.t - self.t_start
          v_loss_factor /= self.t - self.t_start
  
      # Batch re-compute for efficient backprop
      batch_obs = self.batch_states(
          [self.past_obs[i] for i in range(self.t_start, self.t)],
          self.device,
          self.phi,
      )
      if self.recurrent:
          (batch_distrib, batch_v), _ = pack_and_forward(
              self.model,
              [batch_obs],
              self.past_recurrent_state[self.t_start],
          )
      else:
          batch_distrib, batch_v = self.model(batch_obs)
      batch_action = torch.stack(
          [self.past_action[i] for i in range(self.t_start, self.t)]
      )
      batch_log_prob = batch_distrib.log_prob(batch_action)
      batch_entropy = batch_distrib.entropy()
      rev_returns = []
      for i in reversed(range(self.t_start, self.t)):
          R *= self.gamma
          R += self.past_rewards[i]
          rev_returns.append(R)
      batch_return = torch.as_tensor(
          list(reversed(rev_returns)), dtype=torch.float
      )
      batch_adv = batch_return - batch_v.detach().squeeze(-1)
      assert batch_log_prob.shape == (n,)
      assert batch_adv.shape == (n,)
      assert batch_entropy.shape == (n,)
      pi_loss = torch.sum(
          -batch_adv * batch_log_prob - self.beta * batch_entropy, dim=0
      )
      assert batch_v.shape == (n, 1)
      assert batch_return.shape == (n,)
      v_loss = (
          F.mse_loss(batch_v, batch_return[..., None], reduction="sum")
          / 2
      )
  
      if pi_loss_factor != 1.0:
          pi_loss *= pi_loss_factor
  
      if v_loss_factor != 1.0:
          v_loss *= v_loss_factor
  
      if self.process_idx == 0:
          logger.debug("pi_loss:%s v_loss:%s", pi_loss, v_loss)
  
      total_loss = torch.squeeze(pi_loss) + torch.squeeze(v_loss)
  
      # Compute gradients using thread-specific model
      self.model.zero_grad()
      total_loss.backward()
      if self.max_grad_norm is not None:
          clip_l2_grad_norm_(
              self.model.parameters(), self.max_grad_norm
          )
      # Copy the gradients to the globally shared model
      copy_param.copy_grad(
          target_link=self.shared_model, source_link=self.model
      )
      # Update the globally shared model
      self.optimizer.step()
      if self.process_idx == 0:
          logger.debug("update")
  
      self.sync_parameters()
  
      self.past_obs = {}
      self.past_action = {}
      self.past_rewards = {}
      self.past_recurrent_state = {}
  
      self.t_start = self.t
  
imports: import copy
  from logging import getLogger
  import torch
  import torch.nn.functional
  import pfrl
  from pfrl import agent
  from pfrl.utils import clip_l2_grad_norm_, copy_param
  from pfrl.utils.batch_states import batch_states
  from pfrl.utils.mode_of_distribution import mode_of_distribution
  from pfrl.utils.recurrent import one_step_forward, pack_and_forward
label_desc: Use image node_id 5 for calling the A3C obj's underlying member method code with example usage: obj.update(statevar) without return types