docstr: input:
  batch: dict(o, s, u, r, u_onehot, avail_u, padded, isover, actor_inputs, critic_inputs)
  epsilon (float): e-greedy discount
  q_values: Q value of eval critic network, shape (ep_num, tr_num, n_agents, n_actions)
docstr_tok: ["input",":","batch",":","dict","(","o",",","s",",","u",",","r",",","u_onehot",",","avail_u",",","padded",",","isover",",","actor_inputs",",","critic_inputs",")","epsilon","(","float",")",":","e-greedy","discount","q_values",":","Q","value","of","eval","critic","network",",","shape","(","ep_num",",","tr_num",",","n_agents",",","n_actions",")"]
code: def _actor_learn(self, batch, epsilon, q_values):
      """input:
      batch: dict(o, s, u, r, u_onehot, avail_u, padded, isover, actor_inputs, critic_inputs)
      epsilon (float): e-greedy discount
      q_values: Q value of eval critic network, shape (ep_num, tr_num, n_agents, n_actions)
      """
      action_prob = self._get_actor_output(batch, epsilon)  # prob of u
  
      # mask: used to compute TD-error, filling data should not affect learning
      u = batch["u"]
      mask = (
          (1 - batch["padded"].float())
          .repeat(1, 1, self.n_agents)
          .to(self.device)
      )  # shape (ep_num, tr_num, 3)
  
      q_taken = torch.gather(q_values, dim=3, index=u).squeeze(
          3
      )  # Q(u_a)
      pi_taken = torch.gather(action_prob, dim=3, index=u).squeeze(
          3
      )  # prob of act that agent a choosen
      pi_taken[mask == 0] = 1.0  # prevent log overflow
      log_pi_taken = torch.log(pi_taken)
  
      # advantage
      baseline = (
          (q_values * action_prob)
          .sum(dim=3, keepdim=True)
          .squeeze(3)
          .detach()
      )
      advantage = (q_taken - baseline).detach()
      loss = -((advantage * log_pi_taken) * mask).sum() / mask.sum()
      self.actor_optimizer.zero_grad()
      loss.backward()
      torch.nn.utils.clip_grad_norm_(
          self.actor_parameters, self.grad_norm_clip
      )
      self.actor_optimizer.step()
  
file_path: PARL/parl/algorithms/torch/coma.py
imports: import torch
  import os
  from copy import deepcopy
  import parl
  from parl.utils.utils import check_model_method
  import numpy
label_desc: Use image node_id 8 for calling the COMA obj's underlying member method code with example usage: obj._actor_learn(batch, epsilon, q_values) without return types