docstr: input:
      batch: dict(o, s, u, r, u_onehot, avail_u, padded, isover, actor_inputs, critic_inputs)
  output:
      q_values: Q value of eval critic network, shape (ep_num, tr_num, n_agents, n_actions)
docstr_tok: ["input",":","batch",":","dict","(","o",",","s",",","u",",","r",",","u_onehot",",","avail_u",",","padded",",","isover",",","actor_inputs",",","critic_inputs",")","output",":","q_values",":","Q","value","of","eval","critic","network",",","shape","(","ep_num",",","tr_num",",","n_agents",",","n_actions",")"]
code: def _critic_learn(self, batch):
      """input:
          batch: dict(o, s, u, r, u_onehot, avail_u, padded, isover, actor_inputs, critic_inputs)
      output:
          q_values: Q value of eval critic network, shape (ep_num, tr_num, n_agents, n_actions)
      """
      u = batch["u"]  # shape (ep_num, tr_num, agent, n_actions)
      u_next = torch.zeros_like(u, dtype=torch.long)
      u_next[:, :-1] = u[:, 1:]
      mask = (
          (1 - batch["padded"].float())
          .repeat(1, 1, self.n_agents)
          .to(self.device)
      )
  
      # get q value for every agent and every action, shape (ep_num, tr_num, n_agents, n_actions)
      q_evals, q_next_target = self._get_critic_output(batch)
      q_values = q_evals.clone()  # used for function return
  
      # get q valur for every agent
      q_evals = torch.gather(q_evals, dim=3, index=u).squeeze(3)
      q_next_target = torch.gather(
          q_next_target, dim=3, index=u_next
      ).squeeze(3)
  
      targets = self._cal_td_target(batch, q_next_target)
  
      td_error = targets.detach() - q_evals
      masked_td_error = mask * td_error  # mask padded data
  
      loss = (
          masked_td_error**2
      ).sum() / mask.sum()  # mask.sum: avail transition num
  
      self.critic_optimizer.zero_grad()
      loss.backward()
      torch.nn.utils.clip_grad_norm_(
          self.critic_parameters, self.grad_norm_clip
      )
      self.critic_optimizer.step()
      return q_values
  
file_path: PARL/parl/algorithms/torch/coma.py
imports: import torch
  import os
  from copy import deepcopy
  import parl
  from parl.utils.utils import check_model_method
  import numpy
label_desc: Use image node_id 7 for calling the COMA obj's underlying member method code with example usage: obj._critic_learn(batch) and returns: q_values