class_docstr: Proximal Policy Optimization algorithm (GPU version).
  Paper: https://arxiv.org/abs/1707.06347
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) Discount factor
  :param n_steps: (int) The number of steps to run for each environment per update
      (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
  :param ent_coef: (float) Entropy coefficient for the loss calculation
  :param learning_rate: (float or callable) The learning rate, it can be a function
  :param vf_coef: (float) Value function coefficient for the loss calculation
  :param max_grad_norm: (float) The maximum value for the gradient clipping
  :param lam: (float) Factor for trade-off of bias vs variance for Generalized Advantage Estimator
  :param nminibatches: (int) Number of training minibatches per update. For recurrent policies,
      the number of environments run in parallel should be a multiple of nminibatches.
  :param noptepochs: (int) Number of epoch when optimizing the surrogate
  :param cliprange: (float or callable) Clipping parameter, it can be a function
  :param cliprange_vf: (float or callable) Clipping parameter for the value function, it can be a function.
      This is a parameter specific to the OpenAI implementation. If None is passed (default),
      then `cliprange` (that is used for the policy) will be used.
      IMPORTANT: this clipping depends on the reward scaling.
      To deactivate value function clipping (and recover the original PPO implementation),
      you have to pass a negative value (e.g. -1).
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
class_docstr_tok: ["Proximal","Policy","Optimization","algorithm","(","GPU","version",")",".","Paper",":","https",":","\/\/arxiv.org\/abs\/1707.06347",":","param","policy",":","(","ActorCriticPolicy","or","str",")","The","policy","model","to","use","(","MlpPolicy",",","CnnPolicy",",","CnnLstmPolicy",",","...",")",":","param","env",":","(","Gym","environment","or","str",")","The","environment","to","learn","from","(","if","registered","in","Gym",",","can","be","str",")",":","param","gamma",":","(","float",")","Discount","factor",":","param","n_steps",":","(","int",")","The","number","of","steps","to","run","for","each","environment","per","update","(","i.e",".","batch","size","is","n_steps","*","n_env","where","n_env","is","number","of","environment","copies","running","in","parallel",")",":","param","ent_coef",":","(","float",")","Entropy","coefficient","for","the","loss","calculation",":","param","learning_rate",":","(","float","or","callable",")","The","learning","rate",",","it","can","be","a","function",":","param","vf_coef",":","(","float",")","Value","function","coefficient","for","the","loss","calculation",":","param","max_grad_norm",":","(","float",")","The","maximum","value","for","the","gradient","clipping",":","param","lam",":","(","float",")","Factor","for","trade-off","of","bias","vs","variance","for","Generalized","Advantage","Estimator",":","param","nminibatches",":","(","int",")","Number","of","training","minibatches","per","update",".","For","recurrent","policies",",","the","number","of","environments","run","in","parallel","should","be","a","multiple","of","nminibatches",".",":","param","noptepochs",":","(","int",")","Number","of","epoch","when","optimizing","the","surrogate",":","param","cliprange",":","(","float","or","callable",")","Clipping","parameter",",","it","can","be","a","function",":","param","cliprange_vf",":","(","float","or","callable",")","Clipping","parameter","for","the","value","function",",","it","can","be","a","function",".","This","is","a","parameter","specific","to","the","OpenAI","implementation",".","If","None","is","passed","(","default",")",",","then","`","cliprange","`","(","that","is","used","for","the","policy",")","will","be","used",".","IMPORTANT",":","this","clipping","depends","on","the","reward","scaling",".","To","deactivate","value","function","clipping","(","and","recover","the","original","PPO","implementation",")",",","you","have","to","pass","a","negative","value","(","e.g",".","-1",")",".",":","param","verbose",":","(","int",")","the","verbosity","level",":","0","none",",","1","training","information",",","2","tensorflow","debug",":","param","tensorboard_log",":","(","str",")","the","log","location","for","tensorboard","(","if","None",",","no","logging",")",":","param","_init_setup_model",":","(","bool",")","Whether","or","not","to","build","the","network","at","the","creation","of","the","instance",":","param","policy_kwargs",":","(","dict",")","additional","arguments","to","be","passed","to","the","policy","on","creation",":","param","full_tensorboard_log",":","(","bool",")","enable","additional","logging","when","using","tensorboard","WARNING",":","this","logging","can","take","a","lot","of","space","quickly",":","param","seed",":","(","int",")","Seed","for","the","pseudo-random","generators","(","python",",","numpy",",","tensorflow",")",".","If","None","(","default",")",",","use","random","seed",".","Note","that","if","you","want","completely","deterministic","results",",","you","must","set","`","n_cpu_tf_sess","`","to","1.",":","param","n_cpu_tf_sess",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","If","None",",","the","number","of","cpu","of","the","current","machine","will","be","used","."]
docstr: Training of PPO2 Algorithm
  
  :param learning_rate: (float) learning rate
  :param cliprange: (float) Clipping factor
  :param obs: (np.ndarray) The current observation of the environment
  :param returns: (np.ndarray) the rewards
  :param masks: (np.ndarray) The last masks for done episodes (used in recurent policies)
  :param actions: (np.ndarray) the actions
  :param values: (np.ndarray) the values
  :param neglogpacs: (np.ndarray) Negative Log-likelihood probability of Actions
  :param update: (int) the current step iteration
  :param writer: (TensorFlow Summary.writer) the writer for tensorboard
  :param states: (np.ndarray) For recurrent policies, the internal state of the recurrent model
  :return: policy gradient loss, value function loss, policy entropy,
          approximation of kl divergence, updated clipping range, training update operation
  :param cliprange_vf: (float) Clipping factor for the value function
docstr_tok: ["Training","of","PPO2","Algorithm",":","param","learning_rate",":","(","float",")","learning","rate",":","param","cliprange",":","(","float",")","Clipping","factor",":","param","obs",":","(","np.ndarray",")","The","current","observation","of","the","environment",":","param","returns",":","(","np.ndarray",")","the","rewards",":","param","masks",":","(","np.ndarray",")","The","last","masks","for","done","episodes","(","used","in","recurent","policies",")",":","param","actions",":","(","np.ndarray",")","the","actions",":","param","values",":","(","np.ndarray",")","the","values",":","param","neglogpacs",":","(","np.ndarray",")","Negative","Log-likelihood","probability","of","Actions",":","param","update",":","(","int",")","the","current","step","iteration",":","param","writer",":","(","TensorFlow","Summary.writer",")","the","writer","for","tensorboard",":","param","states",":","(","np.ndarray",")","For","recurrent","policies",",","the","internal","state","of","the","recurrent","model",":","return",":","policy","gradient","loss",",","value","function","loss",",","policy","entropy",",","approximation","of","kl","divergence",",","updated","clipping","range",",","training","update","operation",":","param","cliprange_vf",":","(","float",")","Clipping","factor","for","the","value","function"]
code: def _train_step(
      self,
      learning_rate,
      cliprange,
      obs,
      returns,
      masks,
      actions,
      values,
      neglogpacs,
      update,
      writer,
      states=None,
      cliprange_vf=None,
  ):
      """
      Training of PPO2 Algorithm
  
      :param learning_rate: (float) learning rate
      :param cliprange: (float) Clipping factor
      :param obs: (np.ndarray) The current observation of the environment
      :param returns: (np.ndarray) the rewards
      :param masks: (np.ndarray) The last masks for done episodes (used in recurent policies)
      :param actions: (np.ndarray) the actions
      :param values: (np.ndarray) the values
      :param neglogpacs: (np.ndarray) Negative Log-likelihood probability of Actions
      :param update: (int) the current step iteration
      :param writer: (TensorFlow Summary.writer) the writer for tensorboard
      :param states: (np.ndarray) For recurrent policies, the internal state of the recurrent model
      :return: policy gradient loss, value function loss, policy entropy,
              approximation of kl divergence, updated clipping range, training update operation
      :param cliprange_vf: (float) Clipping factor for the value function
      """
      advs = returns - values
      advs = (advs - advs.mean()) / (advs.std() + 1e-8)
      td_map = {
          self.train_model.obs_ph: obs,
          self.action_ph: actions,
          self.advs_ph: advs,
          self.rewards_ph: returns,
          self.learning_rate_ph: learning_rate,
          self.clip_range_ph: cliprange,
          self.old_neglog_pac_ph: neglogpacs,
          self.old_vpred_ph: values,
      }
      if states is not None:
          td_map[self.train_model.states_ph] = states
          td_map[self.train_model.dones_ph] = masks
  
      if cliprange_vf is not None and cliprange_vf >= 0:
          td_map[self.clip_range_vf_ph] = cliprange_vf
  
      if states is None:
          update_fac = max(
              self.n_batch // self.nminibatches // self.noptepochs, 1
          )
      else:
          update_fac = max(
              self.n_batch
              // self.nminibatches
              // self.noptepochs
              // self.n_steps,
              1,
          )
  
      if writer is not None:
          # run loss backprop with summary, but once every 10 runs save the metadata (memory, compute time, ...)
          if self.full_tensorboard_log and (1 + update) % 10 == 0:
              run_options = tf.RunOptions(
                  trace_level=tf.RunOptions.FULL_TRACE
              )
              run_metadata = tf.RunMetadata()
              (
                  summary,
                  policy_loss,
                  value_loss,
                  policy_entropy,
                  approxkl,
                  clipfrac,
                  _,
              ) = self.sess.run(
                  [
                      self.summary,
                      self.pg_loss,
                      self.vf_loss,
                      self.entropy,
                      self.approxkl,
                      self.clipfrac,
                      self._train,
                  ],
                  td_map,
                  options=run_options,
                  run_metadata=run_metadata,
              )
              writer.add_run_metadata(
                  run_metadata, "step%d" % (update * update_fac)
              )
          else:
              (
                  summary,
                  policy_loss,
                  value_loss,
                  policy_entropy,
                  approxkl,
                  clipfrac,
                  _,
              ) = self.sess.run(
                  [
                      self.summary,
                      self.pg_loss,
                      self.vf_loss,
                      self.entropy,
                      self.approxkl,
                      self.clipfrac,
                      self._train,
                  ],
                  td_map,
              )
          writer.add_summary(summary, (update * update_fac))
      else:
          (
              policy_loss,
              value_loss,
              policy_entropy,
              approxkl,
              clipfrac,
              _,
          ) = self.sess.run(
              [
                  self.pg_loss,
                  self.vf_loss,
                  self.entropy,
                  self.approxkl,
                  self.clipfrac,
                  self._train,
              ],
              td_map,
          )
  
      return policy_loss, value_loss, policy_entropy, approxkl, clipfrac
  
imports: import time
  import gym
  import numpy
  import tensorflow
  from stable_baselines import logger
  from stable_baselines.common import explained_variance, ActorCriticRLModel, tf_util, SetVerbosity, TensorboardWriter
  from stable_baselines.common.runners import AbstractEnvRunner
  from stable_baselines.common.policies import ActorCriticPolicy, RecurrentActorCriticPolicy
  from stable_baselines.common.schedules import get_schedule_fn
  from stable_baselines.common.tf_util import total_episode_reward_logger
  from stable_baselines.common.math_util import safe_mean
label_desc: Use image node_id 5 for calling the PPO2 obj's underlying member method code with example usage: obj._train_step(learning_rate, cliprange, obs, returns, masks, actions, values, neglogpacs, update, writer, states, cliprange_vf) and returns: policy_loss,  value_loss,  policy_entropy,  approxkl,  clipfrac