class_docstr: Proximal Policy Optimization algorithm (MPI version).
  Paper: https://arxiv.org/abs/1707.06347
  
  :param env: (Gym environment or str) The environment to learn from (if registered in Gym, can be str)
  :param policy: (ActorCriticPolicy or str) The policy model to use (MlpPolicy, CnnPolicy, CnnLstmPolicy, ...)
  :param timesteps_per_actorbatch: (int) timesteps per actor per update
  :param clip_param: (float) clipping parameter epsilon
  :param entcoeff: (float) the entropy loss weight
  :param optim_epochs: (float) the optimizer's number of epochs
  :param optim_stepsize: (float) the optimizer's stepsize
  :param optim_batchsize: (int) the optimizer's the batch size
  :param gamma: (float) discount factor
  :param lam: (float) advantage estimation
  :param adam_epsilon: (float) the epsilon value for the adam optimizer
  :param schedule: (str) The type of scheduler for the learning rate update ('linear', 'constant',
      'double_linear_con', 'middle_drop' or 'double_middle_drop')
  :param verbose: (int) the verbosity level: 0 none, 1 training information, 2 tensorflow debug
  :param tensorboard_log: (str) the log location for tensorboard (if None, no logging)
  :param _init_setup_model: (bool) Whether or not to build the network at the creation of the instance
  :param policy_kwargs: (dict) additional arguments to be passed to the policy on creation
  :param full_tensorboard_log: (bool) enable additional logging when using tensorboard
      WARNING: this logging can take a lot of space quickly
  :param seed: (int) Seed for the pseudo-random generators (python, numpy, tensorflow).
      If None (default), use random seed. Note that if you want completely deterministic
      results, you must set `n_cpu_tf_sess` to 1.
  :param n_cpu_tf_sess: (int) The number of threads for TensorFlow operations
      If None, the number of cpu of the current machine will be used.
code: def learn(
      self,
      total_timesteps,
      callback=None,
      log_interval=100,
      tb_log_name="PPO1",
      reset_num_timesteps=True,
  ):
      new_tb_log = self._init_num_timesteps(reset_num_timesteps)
      callback = self._init_callback(callback)
  
      with SetVerbosity(self.verbose), TensorboardWriter(
          self.graph, self.tensorboard_log, tb_log_name, new_tb_log
      ) as writer:
          self._setup_learn()
  
          assert issubclass(self.policy, ActorCriticPolicy), (
              "Error: the input policy for the PPO1 model must be "
              "an instance of common.policies.ActorCriticPolicy."
          )
  
          with self.sess.as_default():
              self.adam.sync()
              callback.on_training_start(locals(), globals())
  
              # Prepare for rollouts
              seg_gen = traj_segment_generator(
                  self.policy_pi,
                  self.env,
                  self.timesteps_per_actorbatch,
                  callback=callback,
              )
  
              episodes_so_far = 0
              timesteps_so_far = 0
              iters_so_far = 0
              t_start = time.time()
  
              # rolling buffer for episode lengths
              len_buffer = deque(maxlen=100)
              # rolling buffer for episode rewards
              reward_buffer = deque(maxlen=100)
  
              while True:
                  if timesteps_so_far >= total_timesteps:
                      break
  
                  if self.schedule == "constant":
                      cur_lrmult = 1.0
                  elif self.schedule == "linear":
                      cur_lrmult = max(
                          1.0
                          - float(timesteps_so_far) / total_timesteps,
                          0,
                      )
                  else:
                      raise NotImplementedError
  
                  logger.log(
                      "********** Iteration %i ************"
                      % iters_so_far
                  )
  
                  seg = seg_gen.__next__()
  
                  # Stop training early (triggered by the callback)
                  if not seg.get(
                      "continue_training", True
                  ):  # pytype: disable=attribute-error
                      break
  
                  add_vtarg_and_adv(seg, self.gamma, self.lam)
  
                  # ob, ac, atarg, ret, td1ret = map(np.concatenate, (obs, acs, atargs, rets, td1rets))
                  observations, actions = (
                      seg["observations"],
                      seg["actions"],
                  )
                  atarg, tdlamret = seg["adv"], seg["tdlamret"]
  
                  # true_rew is the reward without discount
                  if writer is not None:
                      total_episode_reward_logger(
                          self.episode_reward,
                          seg["true_rewards"].reshape(
                              (self.n_envs, -1)
                          ),
                          seg["dones"].reshape((self.n_envs, -1)),
                          writer,
                          self.num_timesteps,
                      )
  
                  # predicted value function before udpate
                  vpredbefore = seg["vpred"]
  
                  # standardized advantage function estimate
                  atarg = (atarg - atarg.mean()) / atarg.std()
                  dataset = Dataset(
                      dict(
                          ob=observations,
                          ac=actions,
                          atarg=atarg,
                          vtarg=tdlamret,
                      ),
                      shuffle=not self.policy.recurrent,
                  )
                  optim_batchsize = (
                      self.optim_batchsize or observations.shape[0]
                  )
  
                  # set old parameter values to new parameter values
                  self.assign_old_eq_new(sess=self.sess)
                  logger.log("Optimizing...")
                  logger.log(fmt_row(13, self.loss_names))
  
                  # Here we do a bunch of optimization epochs over the data
                  for k in range(self.optim_epochs):
                      # list of tuples, each of which gives the loss for a minibatch
                      losses = []
                      for i, batch in enumerate(
                          dataset.iterate_once(optim_batchsize)
                      ):
                          steps = (
                              self.num_timesteps
                              + k * optim_batchsize
                              + int(
                                  i
                                  * (
                                      optim_batchsize
                                      / len(dataset.data_map)
                                  )
                              )
                          )
                          if writer is not None:
                              # run loss backprop with summary, but once every 10 runs save the metadata
                              # (memory, compute time, ...)
                              if (
                                  self.full_tensorboard_log
                                  and (1 + k) % 10 == 0
                              ):
                                  run_options = tf.RunOptions(
                                      trace_level=tf.RunOptions.FULL_TRACE
                                  )
                                  run_metadata = tf.RunMetadata()
                                  (
                                      summary,
                                      grad,
                                      *newlosses,
                                  ) = self.lossandgrad(
                                      batch["ob"],
                                      batch["ob"],
                                      batch["ac"],
                                      batch["atarg"],
                                      batch["vtarg"],
                                      cur_lrmult,
                                      sess=self.sess,
                                      options=run_options,
                                      run_metadata=run_metadata,
                                  )
                                  writer.add_run_metadata(
                                      run_metadata, "step%d" % steps
                                  )
                              else:
                                  (
                                      summary,
                                      grad,
                                      *newlosses,
                                  ) = self.lossandgrad(
                                      batch["ob"],
                                      batch["ob"],
                                      batch["ac"],
                                      batch["atarg"],
                                      batch["vtarg"],
                                      cur_lrmult,
                                      sess=self.sess,
                                  )
                              writer.add_summary(summary, steps)
                          else:
                              _, grad, *newlosses = self.lossandgrad(
                                  batch["ob"],
                                  batch["ob"],
                                  batch["ac"],
                                  batch["atarg"],
                                  batch["vtarg"],
                                  cur_lrmult,
                                  sess=self.sess,
                              )
  
                          self.adam.update(
                              grad, self.optim_stepsize * cur_lrmult
                          )
                          losses.append(newlosses)
                      logger.log(fmt_row(13, np.mean(losses, axis=0)))
  
                  logger.log("Evaluating losses...")
                  losses = []
                  for batch in dataset.iterate_once(optim_batchsize):
                      newlosses = self.compute_losses(
                          batch["ob"],
                          batch["ob"],
                          batch["ac"],
                          batch["atarg"],
                          batch["vtarg"],
                          cur_lrmult,
                          sess=self.sess,
                      )
                      losses.append(newlosses)
                  mean_losses, _, _ = mpi_moments(losses, axis=0)
                  logger.log(fmt_row(13, mean_losses))
                  for loss_val, name in zipsame(
                      mean_losses, self.loss_names
                  ):
                      logger.record_tabular("loss_" + name, loss_val)
                  logger.record_tabular(
                      "ev_tdlam_before",
                      explained_variance(vpredbefore, tdlamret),
                  )
  
                  # local values
                  lrlocal = (seg["ep_lens"], seg["ep_rets"])
  
                  # list of tuples
                  listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal)
                  lens, rews = map(flatten_lists, zip(*listoflrpairs))
                  len_buffer.extend(lens)
                  reward_buffer.extend(rews)
                  if len(len_buffer) > 0:
                      logger.record_tabular(
                          "EpLenMean", np.mean(len_buffer)
                      )
                      logger.record_tabular(
                          "EpRewMean", np.mean(reward_buffer)
                      )
                  logger.record_tabular("EpThisIter", len(lens))
                  episodes_so_far += len(lens)
                  current_it_timesteps = MPI.COMM_WORLD.allreduce(
                      seg["total_timestep"]
                  )
                  timesteps_so_far += current_it_timesteps
                  self.num_timesteps += current_it_timesteps
                  iters_so_far += 1
                  logger.record_tabular(
                      "EpisodesSoFar", episodes_so_far
                  )
                  logger.record_tabular(
                      "TimestepsSoFar", self.num_timesteps
                  )
                  logger.record_tabular(
                      "TimeElapsed", time.time() - t_start
                  )
                  if (
                      self.verbose >= 1
                      and MPI.COMM_WORLD.Get_rank() == 0
                  ):
                      logger.dump_tabular()
      callback.on_training_end()
      return self
  
imports: import time
  from collections import deque
  import gym
  import numpy
  import tensorflow
  from mpi4py import MPI
  from stable_baselines.common import Dataset, explained_variance, fmt_row, zipsame, ActorCriticRLModel, SetVerbosity, TensorboardWriter
  from stable_baselines import logger
  import stable_baselines.common.tf_util
  from stable_baselines.common.tf_util import total_episode_reward_logger
  from stable_baselines.common.policies import ActorCriticPolicy
  from stable_baselines.common.mpi_adam import MpiAdam
  from stable_baselines.common.mpi_moments import mpi_moments
  from stable_baselines.common.misc_util import flatten_lists
  from stable_baselines.common.runners import traj_segment_generator
  from stable_baselines.trpo_mpi.utils import add_vtarg_and_adv
label_desc: Use image node_id 4 for calling the PPO1 obj's underlying member method code with example usage: obj.learn(total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps) and returns: self