class_docstr: Trust Region Policy Optimization (https://arxiv.org/abs/1502.05477)
  
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param gamma: (float) the discount value
  :param timesteps_per_batch: (int) the number of timesteps to run per batch (horizon)
  :param max_kl: (float) the Kullback-Leibler loss threshold
  :param cg_iters: (int) the number of iterations for the conjugate gradient calculation
  :param lam: (float) GAE factor
  :param entcoeff: (float) the weight for the entropy loss
  :param cg_damping: (float) the compute gradient dampening factor
  :param vf_stepsize: (float) the value function stepsize
  :param vf_iters: (int) the value function's number iterations for learning
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
code: def setup_model(self):
      # prevent import loops
      from stable_baselines.gail.adversary import TransitionClassifier
  
      with SetVerbosity(self.verbose):
          assert issubclass(self.policy, ActorCriticPolicy), (
              "Error: the input policy for the TRPO model must be "
              "an instance of common.policies.ActorCriticPolicy."
          )
  
          self.nworkers = MPI.COMM_WORLD.Get_size()
          self.rank = MPI.COMM_WORLD.Get_rank()
          np.set_printoptions(precision=3)
  
          self.graph = tf.Graph()
          with self.graph.as_default():
              self.set_random_seed(self.seed)
              self.sess = tf_util.make_session(
                  num_cpu=self.n_cpu_tf_sess, graph=self.graph
              )
  
              if self.using_gail:
                  self.reward_giver = TransitionClassifier(
                      self.observation_space,
                      self.action_space,
                      self.hidden_size_adversary,
                      entcoeff=self.adversary_entcoeff,
                  )
  
              # Construct network for new policy
              self.policy_pi = self.policy(
                  self.sess,
                  self.observation_space,
                  self.action_space,
                  self.n_envs,
                  1,
                  None,
                  reuse=False,
                  **self.policy_kwargs,
              )
  
              # Network for old policy
              with tf.variable_scope("oldpi", reuse=False):
                  old_policy = self.policy(
                      self.sess,
                      self.observation_space,
                      self.action_space,
                      self.n_envs,
                      1,
                      None,
                      reuse=False,
                      **self.policy_kwargs,
                  )
  
              with tf.variable_scope("loss", reuse=False):
                  atarg = tf.placeholder(
                      dtype=tf.float32, shape=[None]
                  )  # Target advantage function (if applicable)
                  ret = tf.placeholder(
                      dtype=tf.float32, shape=[None]
                  )  # Empirical return
  
                  observation = self.policy_pi.obs_ph
                  action = self.policy_pi.pdtype.sample_placeholder(
                      [None]
                  )
  
                  kloldnew = old_policy.proba_distribution.kl(
                      self.policy_pi.proba_distribution
                  )
                  ent = self.policy_pi.proba_distribution.entropy()
                  meankl = tf.reduce_mean(kloldnew)
                  meanent = tf.reduce_mean(ent)
                  entbonus = self.entcoeff * meanent
  
                  vferr = tf.reduce_mean(
                      tf.square(self.policy_pi.value_flat - ret)
                  )
  
                  # advantage * pnew / pold
                  ratio = tf.exp(
                      self.policy_pi.proba_distribution.logp(action)
                      - old_policy.proba_distribution.logp(action)
                  )
                  surrgain = tf.reduce_mean(ratio * atarg)
  
                  optimgain = surrgain + entbonus
                  losses = [
                      optimgain,
                      meankl,
                      entbonus,
                      surrgain,
                      meanent,
                  ]
                  self.loss_names = [
                      "optimgain",
                      "meankl",
                      "entloss",
                      "surrgain",
                      "entropy",
                  ]
  
                  dist = meankl
  
                  all_var_list = tf_util.get_trainable_vars("model")
                  var_list = [
                      v
                      for v in all_var_list
                      if "/vf" not in v.name and "/q/" not in v.name
                  ]
                  vf_var_list = [
                      v
                      for v in all_var_list
                      if "/pi" not in v.name and "/logstd" not in v.name
                  ]
  
                  self.get_flat = tf_util.GetFlat(
                      var_list, sess=self.sess
                  )
                  self.set_from_flat = tf_util.SetFromFlat(
                      var_list, sess=self.sess
                  )
  
                  klgrads = tf.gradients(dist, var_list)
                  flat_tangent = tf.placeholder(
                      dtype=tf.float32, shape=[None], name="flat_tan"
                  )
                  shapes = [
                      var.get_shape().as_list() for var in var_list
                  ]
                  start = 0
                  tangents = []
                  for shape in shapes:
                      var_size = tf_util.intprod(shape)
                      tangents.append(
                          tf.reshape(
                              flat_tangent[start : start + var_size],
                              shape,
                          )
                      )
                      start += var_size
                  gvp = tf.add_n(
                      [
                          tf.reduce_sum(grad * tangent)
                          for (grad, tangent) in zipsame(
                              klgrads, tangents
                          )
                      ]
                  )  # pylint: disable=E1111
                  # Fisher vector products
                  fvp = tf_util.flatgrad(gvp, var_list)
  
                  tf.summary.scalar("entropy_loss", meanent)
                  tf.summary.scalar("policy_gradient_loss", optimgain)
                  tf.summary.scalar("value_function_loss", surrgain)
                  tf.summary.scalar(
                      "approximate_kullback-leibler", meankl
                  )
                  tf.summary.scalar(
                      "loss",
                      optimgain
                      + meankl
                      + entbonus
                      + surrgain
                      + meanent,
                  )
  
                  self.assign_old_eq_new = tf_util.function(
                      [],
                      [],
                      updates=[
                          tf.assign(oldv, newv)
                          for (oldv, newv) in zipsame(
                              tf_util.get_globals_vars("oldpi"),
                              tf_util.get_globals_vars("model"),
                          )
                      ],
                  )
                  self.compute_losses = tf_util.function(
                      [observation, old_policy.obs_ph, action, atarg],
                      losses,
                  )
                  self.compute_fvp = tf_util.function(
                      [
                          flat_tangent,
                          observation,
                          old_policy.obs_ph,
                          action,
                          atarg,
                      ],
                      fvp,
                  )
                  self.compute_vflossandgrad = tf_util.function(
                      [observation, old_policy.obs_ph, ret],
                      tf_util.flatgrad(vferr, vf_var_list),
                  )
  
                  @contextmanager
                  def timed(msg):
                      if self.rank == 0 and self.verbose >= 1:
                          print(colorize(msg, color="magenta"))
                          start_time = time.time()
                          yield
                          print(
                              colorize(
                                  "done in {:.3f} seconds".format(
                                      (time.time() - start_time)
                                  ),
                                  color="magenta",
                              )
                          )
                      else:
                          yield
  
                  def allmean(arr):
                      assert isinstance(arr, np.ndarray)
                      out = np.empty_like(arr)
                      MPI.COMM_WORLD.Allreduce(arr, out, op=MPI.SUM)
                      out /= self.nworkers
                      return out
  
                  tf_util.initialize(sess=self.sess)
  
                  th_init = self.get_flat()
                  MPI.COMM_WORLD.Bcast(th_init, root=0)
                  self.set_from_flat(th_init)
  
              with tf.variable_scope("Adam_mpi", reuse=False):
                  self.vfadam = MpiAdam(vf_var_list, sess=self.sess)
                  if self.using_gail:
                      self.d_adam = MpiAdam(
                          self.reward_giver.get_trainable_variables(),
                          sess=self.sess,
                      )
                      self.d_adam.sync()
                  self.vfadam.sync()
  
              with tf.variable_scope("input_info", reuse=False):
                  tf.summary.scalar(
                      "discounted_rewards", tf.reduce_mean(ret)
                  )
                  tf.summary.scalar(
                      "learning_rate", tf.reduce_mean(self.vf_stepsize)
                  )
                  tf.summary.scalar("advantage", tf.reduce_mean(atarg))
                  tf.summary.scalar(
                      "kl_clip_range", tf.reduce_mean(self.max_kl)
                  )
  
                  if self.full_tensorboard_log:
                      tf.summary.histogram("discounted_rewards", ret)
                      tf.summary.histogram(
                          "learning_rate", self.vf_stepsize
                      )
                      tf.summary.histogram("advantage", atarg)
                      tf.summary.histogram("kl_clip_range", self.max_kl)
                      if tf_util.is_image(self.observation_space):
                          tf.summary.image("observation", observation)
                      else:
                          tf.summary.histogram(
                              "observation", observation
                          )
  
              self.timed = timed
              self.allmean = allmean
  
              self.step = self.policy_pi.step
              self.proba_step = self.policy_pi.proba_step
              self.initial_state = self.policy_pi.initial_state
  
              self.params = tf_util.get_trainable_vars(
                  "model"
              ) + tf_util.get_trainable_vars("oldpi")
              if self.using_gail:
                  self.params.extend(
                      self.reward_giver.get_trainable_variables()
                  )
  
              self.summary = tf.summary.merge_all()
  
              self.compute_lossandgrad = tf_util.function(
                  [observation, old_policy.obs_ph, action, atarg, ret],
                  [self.summary, tf_util.flatgrad(optimgain, var_list)]
                  + losses,
              )
  
imports: import time
  from contextlib import contextmanager
  from collections import deque
  import gym
  from mpi4py import MPI
  import tensorflow
  import numpy
  import stable_baselines.common.tf_util
  from stable_baselines.common.tf_util import total_episode_reward_logger
  from stable_baselines.common import explained_variance, zipsame, dataset, fmt_row, colorize, ActorCriticRLModel, SetVerbosity, TensorboardWriter
  from stable_baselines import logger
  from stable_baselines.common.mpi_adam import MpiAdam
  from stable_baselines.common.cg import conjugate_gradient
  from stable_baselines.common.policies import ActorCriticPolicy
  from stable_baselines.common.misc_util import flatten_lists
  from stable_baselines.common.runners import traj_segment_generator
  from stable_baselines.trpo_mpi.utils import add_vtarg_and_adv
label_desc: Use image node_id 3 for calling the TRPO obj's underlying member method code with example usage: obj.setup_model() and returns: out