docstr: Update rule for ppo algorithm
  Args:
      obs_batch (paddle.tensor): a batch of states
      actions_batch (paddle.tensor): a batch of actions
      value_preds_batch (paddle.tensor): a batch of predicted state value
      return_batch (paddle.tensor): a batch of discounted return
      old_action_log_probs_batch (paddle.tensor): a batch of log prob of old actions
      adv_targ (paddle.tensor): a batch of advantage value
docstr_tok: ["Update","rule","for","ppo","algorithm","Args",":","obs_batch","(","paddle.tensor",")",":","a","batch","of","states","actions_batch","(","paddle.tensor",")",":","a","batch","of","actions","value_preds_batch","(","paddle.tensor",")",":","a","batch","of","predicted","state","value","return_batch","(","paddle.tensor",")",":","a","batch","of","discounted","return","old_action_log_probs_batch","(","paddle.tensor",")",":","a","batch","of","log","prob","of","old","actions","adv_targ","(","paddle.tensor",")",":","a","batch","of","advantage","value"]
code: def learn(
      self,
      obs_batch,
      actions_batch,
      value_preds_batch,
      return_batch,
      old_action_log_probs_batch,
      adv_targ,
  ):
      """Update rule for ppo algorithm
      Args:
          obs_batch (paddle.tensor): a batch of states
          actions_batch (paddle.tensor): a batch of actions
          value_preds_batch (paddle.tensor): a batch of predicted state value
          return_batch (paddle.tensor): a batch of discounted return
          old_action_log_probs_batch (paddle.tensor): a batch of log prob of old actions
          adv_targ (paddle.tensor): a batch of advantage value
      """
  
      values = self.critic(obs_batch)
      probs = self.actor(obs_batch)
  
      dist = Categorical(probs)
      action_log_probs = dist.log_prob(actions_batch)
      dist_entropy = dist.entropy().mean()
  
      ratio = paddle.exp(action_log_probs - old_action_log_probs_batch)
      surr1 = ratio * adv_targ
      surr2 = (
          paddle.clip(
              ratio, 1.0 - self.clip_param, 1.0 + self.clip_param
          )
          * adv_targ
      )
      action_loss = -paddle.minimum(surr1, surr2).mean()
  
      if self.use_clipped_value_loss:
          value_pred_clipped = value_preds_batch + (
              values - value_preds_batch
          ).clip(-self.clip_param, self.clip_param)
          value_losses = (values - return_batch).pow(2)
          value_losses_clipped = (
              value_pred_clipped - return_batch
          ).pow(2)
          value_loss = (
              0.5
              * paddle.maximum(
                  value_losses, value_losses_clipped
              ).mean()
          )
      else:
          value_loss = 0.5 * (return_batch - values).pow(2).mean()
  
      self.optim.clear_grad()
      (
          value_loss + action_loss - dist_entropy * self.entropy_coef
      ).backward()
      self.optim.step()
  
      return value_loss.item(), action_loss.item(), dist_entropy.item()
  
file_path: PARL/examples/Baselines/Halite_competition/paddle/rl_trainer/algorithm.py
imports: import paddle
  import paddle.nn
  import paddle.optimizer
  from paddle.distribution import Categorical
label_desc: Use image node_id 2 for calling the PPO obj's underlying member method code with example usage: obj.learn(obs_batch, actions_batch, value_preds_batch, return_batch, old_action_log_probs_batch, adv_targ) and returns: value_loss,  action_loss,  dist_entropy