class_docstr: PPO class
class_docstr_tok: ["PPO","class"]
docstr: Update policy network
  :param state: state batch
  :param action: action batch
  :param adv: advantage batch
  :param old_pi: old pi distribution
  :return: kl_mean or None
docstr_tok: ["Update","policy","network",":","param","state",":","state","batch",":","param","action",":","action","batch",":","param","adv",":","advantage","batch",":","param","old_pi",":","old","pi","distribution",":","return",":","kl_mean","or","None"]
code: def train_actor(self, state, action, adv, old_pi):
      """
      Update policy network
      :param state: state batch
      :param action: action batch
      :param adv: advantage batch
      :param old_pi: old pi distribution
      :return: kl_mean or None
      """
      with tf.GradientTape() as tape:
          mean, std = self.actor(state), tf.exp(self.actor.logstd)
          pi = tfp.distributions.Normal(mean, std)
  
          ratio = tf.exp(pi.log_prob(action) - old_pi.log_prob(action))
          surr = ratio * adv
          if self.method == "penalty":  # ppo penalty
              kl = tfp.distributions.kl_divergence(old_pi, pi)
              kl_mean = tf.reduce_mean(kl)
              loss = -(tf.reduce_mean(surr - self.lam * kl))
          else:  # ppo clip
              loss = -tf.reduce_mean(
                  tf.minimum(
                      surr,
                      tf.clip_by_value(
                          ratio, 1.0 - self.epsilon, 1.0 + self.epsilon
                      )
                      * adv,
                  )
              )
      a_gard = tape.gradient(loss, self.actor.trainable_weights)
      self.actor_opt.apply_gradients(
          zip(a_gard, self.actor.trainable_weights)
      )
  
      if self.method == "kl_pen":
          return kl_mean
  
imports: import argparse
  import os
  import queue
  import threading
  import time
  import gym
  import matplotlib.pyplot
  import numpy
  import tensorflow
  import tensorflow_probability
  import tensorlayer
label_desc: Use image node_id 2 for calling the PPO obj's underlying member method code with example usage: obj.train_actor(state, action, adv, old_pi) and returns: kl_mean