class_docstr: Proximal Policy Optimization algorithm (GPU version).
  Paper: https://arxiv.org/abs/1707.06347
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) Discount factor
  :param n_steps: (int) The number of steps to run for each environment per update
      (i.e. batch size is n_steps * n_env where n_env is number of environment copies running in parallel)
  :param ent_coef: (float) Entropy coefficient for the loss calculation
  :param learning_rate: (float or callable) The learning rate, it can be a function
  :param vf_coef: (float) Value function coefficient for the loss calculation
  :param max_grad_norm: (float) The maximum value for the gradient clipping
  :param lam: (float) Factor for trade-off of bias vs variance for Generalized Advantage Estimator
  :param nminibatches: (int) Number of training minibatches per update. For recurrent policies,
      the number of environments run in parallel should be a multiple of nminibatches.
  :param noptepochs: (int) Number of epoch when optimizing the surrogate
  :param cliprange: (float or callable) Clipping parameter, it can be a function
  :param cliprange_vf: (float or callable) Clipping parameter for the value function, it can be a function.
      This is a parameter specific to the OpenAI implementation. If None is passed (default),
      then `cliprange` (that is used for the policy) will be used.
      IMPORTANT: this clipping depends on the reward scaling.
      To deactivate value function clipping (and recover the original PPO implementation),
      you have to pass a negative value (e.g. -1).
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
class_docstr_tok: ["Proximal","Policy","Optimization","algorithm","(","GPU","version",")",".","Paper",":","https",":","\/\/arxiv.org\/abs\/1707.06347",":","param","policy",":","(","ActorCriticPolicy","or","str",")","The","policy","model","to","use","(","MlpPolicy",",","CnnPolicy",",","CnnLstmPolicy",",","...",")",":","param","env",":","(","Gym","environment","or","str",")","The","environment","to","learn","from","(","if","registered","in","Gym",",","can","be","str",")",":","param","gamma",":","(","float",")","Discount","factor",":","param","n_steps",":","(","int",")","The","number","of","steps","to","run","for","each","environment","per","update","(","i.e",".","batch","size","is","n_steps","*","n_env","where","n_env","is","number","of","environment","copies","running","in","parallel",")",":","param","ent_coef",":","(","float",")","Entropy","coefficient","for","the","loss","calculation",":","param","learning_rate",":","(","float","or","callable",")","The","learning","rate",",","it","can","be","a","function",":","param","vf_coef",":","(","float",")","Value","function","coefficient","for","the","loss","calculation",":","param","max_grad_norm",":","(","float",")","The","maximum","value","for","the","gradient","clipping",":","param","lam",":","(","float",")","Factor","for","trade-off","of","bias","vs","variance","for","Generalized","Advantage","Estimator",":","param","nminibatches",":","(","int",")","Number","of","training","minibatches","per","update",".","For","recurrent","policies",",","the","number","of","environments","run","in","parallel","should","be","a","multiple","of","nminibatches",".",":","param","noptepochs",":","(","int",")","Number","of","epoch","when","optimizing","the","surrogate",":","param","cliprange",":","(","float","or","callable",")","Clipping","parameter",",","it","can","be","a","function",":","param","cliprange_vf",":","(","float","or","callable",")","Clipping","parameter","for","the","value","function",",","it","can","be","a","function",".","This","is","a","parameter","specific","to","the","OpenAI","implementation",".","If","None","is","passed","(","default",")",",","then","`","cliprange","`","(","that","is","used","for","the","policy",")","will","be","used",".","IMPORTANT",":","this","clipping","depends","on","the","reward","scaling",".","To","deactivate","value","function","clipping","(","and","recover","the","original","PPO","implementation",")",",","you","have","to","pass","a","negative","value","(","e.g",".","-1",")",".",":","param","verbose",":","(","int",")","the","verbosity","level",":","0","none",",","1","training","information",",","2","tensorflow","debug",":","param","tensorboard_log",":","(","str",")","the","log","location","for","tensorboard","(","if","None",",","no","logging",")",":","param","_init_setup_model",":","(","bool",")","Whether","or","not","to","build","the","network","at","the","creation","of","the","instance",":","param","policy_kwargs",":","(","dict",")","additional","arguments","to","be","passed","to","the","policy","on","creation",":","param","full_tensorboard_log",":","(","bool",")","enable","additional","logging","when","using","tensorboard","WARNING",":","this","logging","can","take","a","lot","of","space","quickly",":","param","seed",":","(","int",")","Seed","for","the","pseudo-random","generators","(","python",",","numpy",",","tensorflow",")",".","If","None","(","default",")",",","use","random","seed",".","Note","that","if","you","want","completely","deterministic","results",",","you","must","set","`","n_cpu_tf_sess","`","to","1.",":","param","n_cpu_tf_sess",":","(","int",")","The","number","of","threads","for","TensorFlow","operations","If","None",",","the","number","of","cpu","of","the","current","machine","will","be","used","."]
code: def setup_model(self):
      with SetVerbosity(self.verbose):
          assert issubclass(self.policy, ActorCriticPolicy), (
              "Error: the input policy for the PPO2 model must be "
              "an instance of common.policies.ActorCriticPolicy."
          )
  
          self.n_batch = self.n_envs * self.n_steps
  
          self.graph = tf.Graph()
          with self.graph.as_default():
              self.set_random_seed(self.seed)
              self.sess = tf_util.make_session(
                  num_cpu=self.n_cpu_tf_sess, graph=self.graph
              )
  
              n_batch_step = None
              n_batch_train = None
              if issubclass(self.policy, RecurrentActorCriticPolicy):
                  assert self.n_envs % self.nminibatches == 0, (
                      "For recurrent policies, "
                      "the number of environments run in parallel should be a multiple of nminibatches."
                  )
                  n_batch_step = self.n_envs
                  n_batch_train = self.n_batch // self.nminibatches
  
              act_model = self.policy(
                  self.sess,
                  self.observation_space,
                  self.action_space,
                  self.n_envs,
                  1,
                  n_batch_step,
                  reuse=False,
                  **self.policy_kwargs,
              )
              with tf.variable_scope(
                  "train_model",
                  reuse=True,
                  custom_getter=tf_util.outer_scope_getter(
                      "train_model"
                  ),
              ):
                  train_model = self.policy(
                      self.sess,
                      self.observation_space,
                      self.action_space,
                      self.n_envs // self.nminibatches,
                      self.n_steps,
                      n_batch_train,
                      reuse=True,
                      **self.policy_kwargs,
                  )
  
              with tf.variable_scope("loss", reuse=False):
                  self.action_ph = (
                      train_model.pdtype.sample_placeholder(
                          [None], name="action_ph"
                      )
                  )
                  self.advs_ph = tf.placeholder(
                      tf.float32, [None], name="advs_ph"
                  )
                  self.rewards_ph = tf.placeholder(
                      tf.float32, [None], name="rewards_ph"
                  )
                  self.old_neglog_pac_ph = tf.placeholder(
                      tf.float32, [None], name="old_neglog_pac_ph"
                  )
                  self.old_vpred_ph = tf.placeholder(
                      tf.float32, [None], name="old_vpred_ph"
                  )
                  self.learning_rate_ph = tf.placeholder(
                      tf.float32, [], name="learning_rate_ph"
                  )
                  self.clip_range_ph = tf.placeholder(
                      tf.float32, [], name="clip_range_ph"
                  )
  
                  neglogpac = train_model.proba_distribution.neglogp(
                      self.action_ph
                  )
                  self.entropy = tf.reduce_mean(
                      train_model.proba_distribution.entropy()
                  )
  
                  vpred = train_model.value_flat
  
                  # Value function clipping: not present in the original PPO
                  if self.cliprange_vf is None:
                      # Default behavior (legacy from OpenAI baselines):
                      # use the same clipping as for the policy
                      self.clip_range_vf_ph = self.clip_range_ph
                      self.cliprange_vf = self.cliprange
                  elif (
                      isinstance(self.cliprange_vf, (float, int))
                      and self.cliprange_vf < 0
                  ):
                      # Original PPO implementation: no value function clipping
                      self.clip_range_vf_ph = None
                  else:
                      # Last possible behavior: clipping range
                      # specific to the value function
                      self.clip_range_vf_ph = tf.placeholder(
                          tf.float32, [], name="clip_range_vf_ph"
                      )
  
                  if self.clip_range_vf_ph is None:
                      # No clipping
                      vpred_clipped = train_model.value_flat
                  else:
                      # Clip the different between old and new value
                      # NOTE: this depends on the reward scaling
                      vpred_clipped = (
                          self.old_vpred_ph
                          + tf.clip_by_value(
                              train_model.value_flat
                              - self.old_vpred_ph,
                              -self.clip_range_vf_ph,
                              self.clip_range_vf_ph,
                          )
                      )
  
                  vf_losses1 = tf.square(vpred - self.rewards_ph)
                  vf_losses2 = tf.square(
                      vpred_clipped - self.rewards_ph
                  )
                  self.vf_loss = 0.5 * tf.reduce_mean(
                      tf.maximum(vf_losses1, vf_losses2)
                  )
  
                  ratio = tf.exp(self.old_neglog_pac_ph - neglogpac)
                  pg_losses = -self.advs_ph * ratio
                  pg_losses2 = -self.advs_ph * tf.clip_by_value(
                      ratio,
                      1.0 - self.clip_range_ph,
                      1.0 + self.clip_range_ph,
                  )
                  self.pg_loss = tf.reduce_mean(
                      tf.maximum(pg_losses, pg_losses2)
                  )
                  self.approxkl = 0.5 * tf.reduce_mean(
                      tf.square(neglogpac - self.old_neglog_pac_ph)
                  )
                  self.clipfrac = tf.reduce_mean(
                      tf.cast(
                          tf.greater(
                              tf.abs(ratio - 1.0), self.clip_range_ph
                          ),
                          tf.float32,
                      )
                  )
                  loss = (
                      self.pg_loss
                      - self.entropy * self.ent_coef
                      + self.vf_loss * self.vf_coef
                  )
  
                  tf.summary.scalar("entropy_loss", self.entropy)
                  tf.summary.scalar(
                      "policy_gradient_loss", self.pg_loss
                  )
                  tf.summary.scalar("value_function_loss", self.vf_loss)
                  tf.summary.scalar(
                      "approximate_kullback-leibler", self.approxkl
                  )
                  tf.summary.scalar("clip_factor", self.clipfrac)
                  tf.summary.scalar("loss", loss)
  
                  with tf.variable_scope("model"):
                      self.params = tf.trainable_variables()
                      if self.full_tensorboard_log:
                          for var in self.params:
                              tf.summary.histogram(var.name, var)
                  grads = tf.gradients(loss, self.params)
                  if self.max_grad_norm is not None:
                      grads, _grad_norm = tf.clip_by_global_norm(
                          grads, self.max_grad_norm
                      )
                  grads = list(zip(grads, self.params))
              trainer = tf.train.AdamOptimizer(
                  learning_rate=self.learning_rate_ph, epsilon=1e-5
              )
              self._train = trainer.apply_gradients(grads)
  
              self.loss_names = [
                  "policy_loss",
                  "value_loss",
                  "policy_entropy",
                  "approxkl",
                  "clipfrac",
              ]
  
              with tf.variable_scope("input_info", reuse=False):
                  tf.summary.scalar(
                      "discounted_rewards",
                      tf.reduce_mean(self.rewards_ph),
                  )
                  tf.summary.scalar(
                      "learning_rate",
                      tf.reduce_mean(self.learning_rate_ph),
                  )
                  tf.summary.scalar(
                      "advantage", tf.reduce_mean(self.advs_ph)
                  )
                  tf.summary.scalar(
                      "clip_range", tf.reduce_mean(self.clip_range_ph)
                  )
                  if self.clip_range_vf_ph is not None:
                      tf.summary.scalar(
                          "clip_range_vf",
                          tf.reduce_mean(self.clip_range_vf_ph),
                      )
  
                  tf.summary.scalar(
                      "old_neglog_action_probability",
                      tf.reduce_mean(self.old_neglog_pac_ph),
                  )
                  tf.summary.scalar(
                      "old_value_pred",
                      tf.reduce_mean(self.old_vpred_ph),
                  )
  
                  if self.full_tensorboard_log:
                      tf.summary.histogram(
                          "discounted_rewards", self.rewards_ph
                      )
                      tf.summary.histogram(
                          "learning_rate", self.learning_rate_ph
                      )
                      tf.summary.histogram("advantage", self.advs_ph)
                      tf.summary.histogram(
                          "clip_range", self.clip_range_ph
                      )
                      tf.summary.histogram(
                          "old_neglog_action_probability",
                          self.old_neglog_pac_ph,
                      )
                      tf.summary.histogram(
                          "old_value_pred", self.old_vpred_ph
                      )
                      if tf_util.is_image(self.observation_space):
                          tf.summary.image(
                              "observation", train_model.obs_ph
                          )
                      else:
                          tf.summary.histogram(
                              "observation", train_model.obs_ph
                          )
  
              self.train_model = train_model
              self.act_model = act_model
              self.step = act_model.step
              self.proba_step = act_model.proba_step
              self.value = act_model.value
              self.initial_state = act_model.initial_state
              tf.global_variables_initializer().run(
                  session=self.sess
              )  # pylint: disable=E1101
  
              self.summary = tf.summary.merge_all()
  
imports: import time
  import gym
  import numpy
  import tensorflow
  from stable_baselines import logger
  from stable_baselines.common import explained_variance, ActorCriticRLModel, tf_util, SetVerbosity, TensorboardWriter
  from stable_baselines.common.runners import AbstractEnvRunner
  from stable_baselines.common.policies import ActorCriticPolicy, RecurrentActorCriticPolicy
  from stable_baselines.common.schedules import get_schedule_fn
  from stable_baselines.common.tf_util import total_episode_reward_logger
  from stable_baselines.common.math_util import safe_mean
label_desc: Use image node_id 4 for calling the PPO2 obj's underlying member method code with example usage: obj.setup_model() without return types