class_docstr: PPO class
class_docstr_tok: ["PPO","class"]
docstr: Update parameter with the constraint of KL divergent
  :return: None
docstr_tok: ["Update","parameter","with","the","constraint","of","KL","divergent",":","return",":","None"]
code: def update(self):
      """
      Update parameter with the constraint of KL divergent
      :return: None
      """
      global GLOBAL_UPDATE_COUNTER
      while not COORD.should_stop():
          if GLOBAL_EP < TRAIN_EPISODES:
              UPDATE_EVENT.wait()  # wait until get batch of data
  
              data = [
                  QUEUE.get() for _ in range(QUEUE.qsize())
              ]  # collect data from all workers
              s, a, r = zip(*data)
              s = np.vstack(s).astype(np.float32)
              a = np.vstack(a).astype(np.float32)
              r = np.vstack(r).astype(np.float32)
              mean, std = self.actor(s), tf.exp(self.actor.logstd)
              pi = tfp.distributions.Normal(mean, std)
              adv = r - self.critic(s)
              # adv = (adv - adv.mean())/(adv.std()+1e-6)     # sometimes helpful
  
              # update actor
              if self.method == "kl_pen":
                  for _ in range(ACTOR_UPDATE_STEPS):
                      kl = self.train_actor(s, a, adv, pi)
                  if kl < self.kl_target / 1.5:
                      self.lam /= 2
                  elif kl > self.kl_target * 1.5:
                      self.lam *= 2
              else:
                  for _ in range(ACTOR_UPDATE_STEPS):
                      self.train_actor(s, a, adv, pi)
  
              # update critic
              for _ in range(CRITIC_UPDATE_STEPS):
                  self.train_critic(r, s)
  
              UPDATE_EVENT.clear()  # updating finished
              GLOBAL_UPDATE_COUNTER = 0  # reset counter
              ROLLING_EVENT.set()
  
imports: import argparse
  import os
  import queue
  import threading
  import time
  import gym
  import matplotlib.pyplot
  import numpy
  import tensorflow
  import tensorflow_probability
  import tensorlayer
label_desc: Use image node_id 4 for calling the PPO obj's underlying member method code with example usage: obj.update() without return types