docstr: Learn policy model with: 
          1. CLIP loss: Clipped Surrogate Objective 
          2. KLPEN loss: Adaptive KL Penalty Objective
      See: https://arxiv.org/pdf/1707.02286.pdf
  
  Args:
      obs: Tensor, (batch_size, obs_dim)
      actions: Tensor, (batch_size, act_dim)
      advantages: Tensor (batch_size, )
      beta: Tensor (1) or None
            if None, use CLIP Loss; else, use KLPEN loss. 
docstr_tok: ["Learn","policy","model","with",":","1",".","CLIP","loss",":","Clipped","Surrogate","Objective","2",".","KLPEN","loss",":","Adaptive","KL","Penalty","Objective","See",":","https",":","\/\/arxiv.org\/pdf\/1707.02286.pdf","Args",":","obs",":","Tensor",",","(","batch_size",",","obs_dim",")","actions",":","Tensor",",","(","batch_size",",","act_dim",")","advantages",":","Tensor","(","batch_size",",",")","beta",":","Tensor","(","1",")","or","None","if","None",",","use","CLIP","Loss",";","else",",","use","KLPEN","loss","."]
code: def policy_learn(self, obs, actions, advantages, beta=None):
      """Learn policy model with:
              1. CLIP loss: Clipped Surrogate Objective
              2. KLPEN loss: Adaptive KL Penalty Objective
          See: https://arxiv.org/pdf/1707.02286.pdf
  
      Args:
          obs: Tensor, (batch_size, obs_dim)
          actions: Tensor, (batch_size, act_dim)
          advantages: Tensor (batch_size, )
          beta: Tensor (1) or None
                if None, use CLIP Loss; else, use KLPEN loss.
      """
      old_means, old_logvars = self.old_policy_model.policy(obs)
      old_means.stop_gradient = True
      old_logvars.stop_gradient = True
      old_logprob = self._calc_logprob(actions, old_means, old_logvars)
  
      means, logvars = self.model.policy(obs)
      logprob = self._calc_logprob(actions, means, logvars)
  
      kl = self._calc_kl(means, logvars, old_means, old_logvars)
      kl = layers.reduce_mean(kl)
  
      if beta is None:  # Clipped Surrogate Objective
          pg_ratio = layers.exp(logprob - old_logprob)
          clipped_pg_ratio = layers.clip(
              pg_ratio, 1 - self.epsilon, 1 + self.epsilon
          )
          surrogate_loss = layers.elementwise_min(
              advantages * pg_ratio, advantages * clipped_pg_ratio
          )
          loss = 0 - layers.reduce_mean(surrogate_loss)
      else:  # Adaptive KL Penalty Objective
          # policy gradient loss
          loss1 = 0 - layers.reduce_mean(
              advantages * layers.exp(logprob - old_logprob)
          )
          # adaptive kl loss
          loss2 = kl * beta
          loss = loss1 + loss2
      optimizer = fluid.optimizer.AdamOptimizer(self.policy_lr)
      optimizer.minimize(loss)
      return loss, kl
  
file_path: PARL/parl/algorithms/fluid/ppo.py
imports: import warnings
  import numpy
  from copy import deepcopy
  from paddle import fluid
  from parl.core.fluid import layers
  from parl.core.fluid.algorithm import Algorithm
label_desc: Use image node_id 6 for calling the PPO obj's underlying member method code with example usage: obj.policy_learn(obs, actions, advantages, beta) and returns: loss,  kl