docstr: Update rule for ppo algorithm
  Args:
      obs_batch (torhc.tensor): a batch of states
      actions_batch (torch.tensor): a batch of actions
      value_preds_batch (torch.tensor): a batch of predicted state value
      return_batch (torch.tensor): a batch of discounted return
      old_action_log_probs_batch (torch.tensor): a batch of log prob of old actions
      adv_targ (torch.tensor): a batch of advantage value
docstr_tok: ["Update","rule","for","ppo","algorithm","Args",":","obs_batch","(","torhc.tensor",")",":","a","batch","of","states","actions_batch","(","torch.tensor",")",":","a","batch","of","actions","value_preds_batch","(","torch.tensor",")",":","a","batch","of","predicted","state","value","return_batch","(","torch.tensor",")",":","a","batch","of","discounted","return","old_action_log_probs_batch","(","torch.tensor",")",":","a","batch","of","log","prob","of","old","actions","adv_targ","(","torch.tensor",")",":","a","batch","of","advantage","value"]
code: def learn(
      self,
      obs_batch,
      actions_batch,
      value_preds_batch,
      return_batch,
      old_action_log_probs_batch,
      adv_targ,
  ):
      """Update rule for ppo algorithm
      Args:
          obs_batch (torhc.tensor): a batch of states
          actions_batch (torch.tensor): a batch of actions
          value_preds_batch (torch.tensor): a batch of predicted state value
          return_batch (torch.tensor): a batch of discounted return
          old_action_log_probs_batch (torch.tensor): a batch of log prob of old actions
          adv_targ (torch.tensor): a batch of advantage value
      """
      values = self.critic(obs_batch)
      probs = self.actor(obs_batch)
      dist = Categorical(probs)
      action_log_probs = dist.log_prob(actions_batch)
      dist_entropy = dist.entropy().mean()
  
      ratio = torch.exp(action_log_probs - old_action_log_probs_batch)
      surr1 = ratio * adv_targ
      surr2 = (
          torch.clamp(
              ratio, 1.0 - self.clip_param, 1.0 + self.clip_param
          )
          * adv_targ
      )
      action_loss = -torch.min(surr1, surr2).mean()
  
      if self.use_clipped_value_loss:
          value_pred_clipped = value_preds_batch + (
              values - value_preds_batch
          ).clamp(-self.clip_param, self.clip_param)
          value_losses = (values - return_batch).pow(2)
          value_losses_clipped = (
              value_pred_clipped - return_batch
          ).pow(2)
          value_loss = (
              0.5 * torch.max(value_losses, value_losses_clipped).mean()
          )
      else:
          value_loss = 0.5 * (return_batch - values).pow(2).mean()
  
      self.critic_optimizer.zero_grad()
      value_loss.backward()
      # nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
      self.critic_optimizer.step()
  
      self.actor_optimizer.zero_grad()
      (action_loss - dist_entropy * self.entropy_coef).backward()
      # nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
      self.actor_optimizer.step()
  
      return value_loss.item(), action_loss.item(), dist_entropy.item()
  
file_path: PARL/examples/Baselines/Halite_competition/torch/rl_trainer/algorithm.py
imports: import torch
  import torch.nn
  import torch.optim
  import parl
  from torch.distributions import Categorical
label_desc: Use image node_id 2 for calling the PPO obj's underlying member method code with example usage: obj.learn(obs_batch, actions_batch, value_preds_batch, return_batch, old_action_log_probs_batch, adv_targ) and returns: value_loss,  action_loss,  dist_entropy