class_docstr: Proximal Policy Optimization algorithm (MPI version).
  Paper: https://arxiv.org/abs/1707.06347
  
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param timesteps_per_actorbatch: (int) timesteps per actor per update
  :param clip_param: (float) clipping parameter epsilon
  :param entcoeff: (float) the entropy loss weight
  :param optim_epochs: (float) the optimizer's number of epochs
  :param optim_stepsize: (float) the optimizer's stepsize
  :param optim_batchsize: (int) the optimizer's the batch size
  :param gamma: (float) discount factor
  :param lam: (float) advantage estimation
  :param adam_epsilon: (float) the epsilon value for the adam optimizer
  :param schedule: (str) The type of scheduler for the learning rate update ('linear', 'constant',
      'double_linear_con', 'middle_drop' or 'double_middle_drop')
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
code: def setup_model(self):
      with SetVerbosity(self.verbose):
          self.graph = tf.Graph()
          with self.graph.as_default():
              self.set_random_seed(self.seed)
              self.sess = tf_util.make_session(
                  num_cpu=self.n_cpu_tf_sess, graph=self.graph
              )
  
              # Construct network for new policy
              self.policy_pi = self.policy(
                  self.sess,
                  self.observation_space,
                  self.action_space,
                  self.n_envs,
                  1,
                  None,
                  reuse=False,
                  **self.policy_kwargs,
              )
  
              # Network for old policy
              with tf.variable_scope("oldpi", reuse=False):
                  old_pi = self.policy(
                      self.sess,
                      self.observation_space,
                      self.action_space,
                      self.n_envs,
                      1,
                      None,
                      reuse=False,
                      **self.policy_kwargs,
                  )
  
              with tf.variable_scope("loss", reuse=False):
                  # Target advantage function (if applicable)
                  atarg = tf.placeholder(dtype=tf.float32, shape=[None])
  
                  # Empirical return
                  ret = tf.placeholder(dtype=tf.float32, shape=[None])
  
                  # learning rate multiplier, updated with schedule
                  lrmult = tf.placeholder(
                      name="lrmult", dtype=tf.float32, shape=[]
                  )
  
                  # Annealed cliping parameter epislon
                  clip_param = self.clip_param * lrmult
  
                  obs_ph = self.policy_pi.obs_ph
                  action_ph = self.policy_pi.pdtype.sample_placeholder(
                      [None]
                  )
  
                  kloldnew = old_pi.proba_distribution.kl(
                      self.policy_pi.proba_distribution
                  )
                  ent = self.policy_pi.proba_distribution.entropy()
                  meankl = tf.reduce_mean(kloldnew)
                  meanent = tf.reduce_mean(ent)
                  pol_entpen = (-self.entcoeff) * meanent
  
                  # pnew / pold
                  ratio = tf.exp(
                      self.policy_pi.proba_distribution.logp(action_ph)
                      - old_pi.proba_distribution.logp(action_ph)
                  )
  
                  # surrogate from conservative policy iteration
                  surr1 = ratio * atarg
                  surr2 = (
                      tf.clip_by_value(
                          ratio, 1.0 - clip_param, 1.0 + clip_param
                      )
                      * atarg
                  )
  
                  # PPO's pessimistic surrogate (L^CLIP)
                  pol_surr = -tf.reduce_mean(tf.minimum(surr1, surr2))
                  vf_loss = tf.reduce_mean(
                      tf.square(self.policy_pi.value_flat - ret)
                  )
                  total_loss = pol_surr + pol_entpen + vf_loss
                  losses = [
                      pol_surr,
                      pol_entpen,
                      vf_loss,
                      meankl,
                      meanent,
                  ]
                  self.loss_names = [
                      "pol_surr",
                      "pol_entpen",
                      "vf_loss",
                      "kl",
                      "ent",
                  ]
  
                  tf.summary.scalar("entropy_loss", pol_entpen)
                  tf.summary.scalar("policy_gradient_loss", pol_surr)
                  tf.summary.scalar("value_function_loss", vf_loss)
                  tf.summary.scalar(
                      "approximate_kullback-leibler", meankl
                  )
                  tf.summary.scalar("clip_factor", clip_param)
                  tf.summary.scalar("loss", total_loss)
  
                  self.params = tf_util.get_trainable_vars("model")
  
                  self.assign_old_eq_new = tf_util.function(
                      [],
                      [],
                      updates=[
                          tf.assign(oldv, newv)
                          for (oldv, newv) in zipsame(
                              tf_util.get_globals_vars("oldpi"),
                              tf_util.get_globals_vars("model"),
                          )
                      ],
                  )
  
              with tf.variable_scope("Adam_mpi", reuse=False):
                  self.adam = MpiAdam(
                      self.params,
                      epsilon=self.adam_epsilon,
                      sess=self.sess,
                  )
  
              with tf.variable_scope("input_info", reuse=False):
                  tf.summary.scalar(
                      "discounted_rewards", tf.reduce_mean(ret)
                  )
                  tf.summary.scalar(
                      "learning_rate",
                      tf.reduce_mean(self.optim_stepsize),
                  )
                  tf.summary.scalar("advantage", tf.reduce_mean(atarg))
                  tf.summary.scalar(
                      "clip_range", tf.reduce_mean(self.clip_param)
                  )
  
                  if self.full_tensorboard_log:
                      tf.summary.histogram("discounted_rewards", ret)
                      tf.summary.histogram(
                          "learning_rate", self.optim_stepsize
                      )
                      tf.summary.histogram("advantage", atarg)
                      tf.summary.histogram(
                          "clip_range", self.clip_param
                      )
                      if tf_util.is_image(self.observation_space):
                          tf.summary.image("observation", obs_ph)
                      else:
                          tf.summary.histogram("observation", obs_ph)
  
              self.step = self.policy_pi.step
              self.proba_step = self.policy_pi.proba_step
              self.initial_state = self.policy_pi.initial_state
  
              tf_util.initialize(sess=self.sess)
  
              self.summary = tf.summary.merge_all()
  
              self.lossandgrad = tf_util.function(
                  [
                      obs_ph,
                      old_pi.obs_ph,
                      action_ph,
                      atarg,
                      ret,
                      lrmult,
                  ],
                  [
                      self.summary,
                      tf_util.flatgrad(total_loss, self.params),
                  ]
                  + losses,
              )
              self.compute_losses = tf_util.function(
                  [
                      obs_ph,
                      old_pi.obs_ph,
                      action_ph,
                      atarg,
                      ret,
                      lrmult,
                  ],
                  losses,
              )
  
imports: import time
  from collections import deque
  import gym
  import numpy
  import tensorflow
  from mpi4py import MPI
  from stable_baselines.common import Dataset, explained_variance, fmt_row, zipsame, ActorCriticRLModel, SetVerbosity, TensorboardWriter
  from stable_baselines import logger
  import stable_baselines.common.tf_util
  from stable_baselines.common.tf_util import total_episode_reward_logger
  from stable_baselines.common.policies import ActorCriticPolicy
  from stable_baselines.common.mpi_adam import MpiAdam
  from stable_baselines.common.mpi_moments import mpi_moments
  from stable_baselines.common.misc_util import flatten_lists
  from stable_baselines.common.runners import traj_segment_generator
  from stable_baselines.trpo_mpi.utils import add_vtarg_and_adv
label_desc: Use image node_id 3 for calling the PPO1 obj's underlying member method code with example usage: obj.setup_model() without return types