docstr: MAPPO algorithm
  
  Args:
      model (parl.Model): model that contains both value network and policy network
      clip_param (float): the clipping strength for value loss clipping
      value_loss_coef (float): the coefficient for value loss
      entropy_coef (float): the coefficient for entropy
      initial_lr (float): initial learning rate.
      huber_delta (float): coefficience of huber loss
      eps (None or float): epsilon for Adam optimizer
      max_grad_norm (float): threshold for grad norm clipping
      use_popart (bool): whether to use PopArt to normalize rewards
      use_value_active_masks (bool): whether to mask useless data in value loss
docstr_tok: ["MAPPO","algorithm","Args",":","model","(","parl.Model",")",":","model","that","contains","both","value","network","and","policy","network","clip_param","(","float",")",":","the","clipping","strength","for","value","loss","clipping","value_loss_coef","(","float",")",":","the","coefficient","for","value","loss","entropy_coef","(","float",")",":","the","coefficient","for","entropy","initial_lr","(","float",")",":","initial","learning","rate",".","huber_delta","(","float",")",":","coefficience","of","huber","loss","eps","(","None","or","float",")",":","epsilon","for","Adam","optimizer","max_grad_norm","(","float",")",":","threshold","for","grad","norm","clipping","use_popart","(","bool",")",":","whether","to","use","PopArt","to","normalize","rewards","use_value_active_masks","(","bool",")",":","whether","to","mask","useless","data","in","value","loss"]
code: def __init__(
      self,
      model,
      clip_param,
      value_loss_coef,
      entropy_coef,
      initial_lr,
      huber_delta,
      eps=None,
      max_grad_norm=None,
      use_popart=True,
      use_value_active_masks=True,
  ):
      """MAPPO algorithm
  
      Args:
          model (parl.Model): model that contains both value network and policy network
          clip_param (float): the clipping strength for value loss clipping
          value_loss_coef (float): the coefficient for value loss
          entropy_coef (float): the coefficient for entropy
          initial_lr (float): initial learning rate.
          huber_delta (float): coefficience of huber loss
          eps (None or float): epsilon for Adam optimizer
          max_grad_norm (float): threshold for grad norm clipping
          use_popart (bool): whether to use PopArt to normalize rewards
          use_value_active_masks (bool): whether to mask useless data in value loss
      """
      self.model = model
      self.device = torch.device(
          "cuda:0" if torch.cuda.is_available() else "cpu"
      )
      self.model.to(self.device)
      self.multi_discrete = self.model.actor.multi_discrete
      self.tpdv = dict(dtype=torch.float32, device=self.device)
  
      self.clip_param = clip_param
      self.value_loss_coef = value_loss_coef
      self.lr = initial_lr
      self.huber_delta = huber_delta
      self.eps = eps
      self.entropy_coef = entropy_coef
      self.max_grad_norm = max_grad_norm
      self._use_popart = use_popart
      self._use_value_active_masks = use_value_active_masks
  
      if self._use_popart:
          self.value_normalizer = PopArt(
              self.model.critic.v_out.weight,
              self.model.critic.v_out.bias,
          )
      else:
          self.value_normalizer = None
  
      self.actor_optimizer = torch.optim.Adam(
          self.model.actor.parameters(), lr=self.lr, eps=self.eps
      )
      self.critic_optimizer = torch.optim.Adam(
          self.model.critic.parameters(), lr=self.lr, eps=self.eps
      )
  
imports: import torch
  import torch.nn
  from torch.distributions import Categorical
  import numpy
  import parl
label_desc: Use image node_id 1 to create a new MAPPO object from inherited base classes: parl with example: obj = MAPPO(model, clip_param, value_loss_coef, entropy_coef, initial_lr, huber_delta, eps, max_grad_norm, use_popart, use_value_active_masks)