docstr: update model with PPO algorithm
  
  Args:
      batch_obs (torch.Tensor):           shape([batch_size] + obs_shape)
      batch_action (torch.Tensor):        shape([batch_size] + action_shape)
      batch_value (torch.Tensor):         shape([batch_size])
      batch_return (torch.Tensor):        shape([batch_size])
      batch_logprob (torch.Tensor):       shape([batch_size])
      batch_adv (torch.Tensor):           shape([batch_size])
      lr (torch.Tensor):
  Returns:
      value_loss (float): value loss
      action_loss (float): policy loss
      entropy_loss (float): entropy loss
docstr_tok: ["update","model","with","PPO","algorithm","Args",":","batch_obs","(","torch.Tensor",")",":","shape","(","[","batch_size","]","+","obs_shape",")","batch_action","(","torch.Tensor",")",":","shape","(","[","batch_size","]","+","action_shape",")","batch_value","(","torch.Tensor",")",":","shape","(","[","batch_size","]",")","batch_return","(","torch.Tensor",")",":","shape","(","[","batch_size","]",")","batch_logprob","(","torch.Tensor",")",":","shape","(","[","batch_size","]",")","batch_adv","(","torch.Tensor",")",":","shape","(","[","batch_size","]",")","lr","(","torch.Tensor",")",":","Returns",":","value_loss","(","float",")",":","value","loss","action_loss","(","float",")",":","policy","loss","entropy_loss","(","float",")",":","entropy","loss"]
code: def learn(
      self,
      batch_obs,
      batch_action,
      batch_value,
      batch_return,
      batch_logprob,
      batch_adv,
      lr=None,
  ):
      """update model with PPO algorithm
  
      Args:
          batch_obs (torch.Tensor):           shape([batch_size] + obs_shape)
          batch_action (torch.Tensor):        shape([batch_size] + action_shape)
          batch_value (torch.Tensor):         shape([batch_size])
          batch_return (torch.Tensor):        shape([batch_size])
          batch_logprob (torch.Tensor):       shape([batch_size])
          batch_adv (torch.Tensor):           shape([batch_size])
          lr (torch.Tensor):
      Returns:
          value_loss (float): value loss
          action_loss (float): policy loss
          entropy_loss (float): entropy loss
      """
      values = self.model.value(batch_obs)
      if self.continuous_action:
          mean, std = self.model.policy(batch_obs)
          dist = Normal(mean, std)
          action_log_probs = dist.log_prob(batch_action).sum(1)
          dist_entropy = dist.entropy().sum(1)
      else:
          logits = self.model.policy(batch_obs)
          dist = Categorical(logits=logits)
  
          act_dim = logits.shape[-1]
          batch_action = paddle.to_tensor(batch_action, dtype="int64")
          actions_onehot = F.one_hot(batch_action, act_dim)
  
          action_log_probs = paddle.sum(
              F.log_softmax(logits) * actions_onehot, axis=-1
          )
          dist_entropy = dist.entropy()
      entropy_loss = dist_entropy.mean()
  
      batch_adv = batch_adv
      if self.norm_adv:
          batch_adv = (batch_adv - batch_adv.mean()) / (
              batch_adv.std() + 1e-8
          )
  
      ratio = paddle.exp(action_log_probs - batch_logprob)
      surr1 = ratio * batch_adv
      surr2 = (
          paddle.clip(
              ratio, 1.0 - self.clip_param, 1.0 + self.clip_param
          )
          * batch_adv
      )
      action_loss = -paddle.minimum(surr1, surr2).mean()
  
      values = values.reshape([-1])
      # calculate value loss using semi gradient TD
      if self.use_clipped_value_loss:
          value_pred_clipped = batch_value + paddle.clip(
              values - batch_value, -self.clip_param, self.clip_param
          )
          value_losses = (values - batch_return).pow(2)
          value_losses_clipped = (
              value_pred_clipped - batch_return
          ).pow(2)
          value_loss = (
              0.5
              * paddle.maximum(
                  value_losses, value_losses_clipped
              ).mean()
          )
      else:
          value_loss = 0.5 * (values - batch_return).pow(2).mean()
  
      loss = (
          value_loss * self.value_loss_coef
          + action_loss
          - entropy_loss * self.entropy_coef
      )
  
      if lr:
          self.optimizer.set_lr(lr)
  
      loss.backward()
      self.optimizer.step()
      self.optimizer.clear_grad()
  
      return value_loss.item(), action_loss.item(), entropy_loss.item()
  
file_path: PARL/parl/algorithms/paddle/ppo.py
imports: import parl
  import paddle
  import paddle.nn
  import paddle.nn.functional
  import paddle.optimizer
  from paddle.distribution import Normal, Categorical
  from parl.utils.utils import check_model_method
label_desc: Use image node_id 2 for calling the PPO obj's underlying member method code with example usage: obj.learn(batch_obs, batch_action, batch_value, batch_return, batch_logprob, batch_adv, lr) and returns: value_loss,  action_loss,  entropy_loss