code: def __init__(
      self,
      policy,
      ob_space,
      ac_space,
      nenvs,
      nsteps,
      ent_coef,
      q_coef,
      gamma,
      max_grad_norm,
      lr,
      rprop_alpha,
      rprop_epsilon,
      total_timesteps,
      lrschedule,
      c,
      trust_region,
      alpha,
      delta,
  ):
      sess = get_session()
      nact = ac_space.n
      nbatch = nenvs * nsteps
  
      A = tf.placeholder(tf.int32, [nbatch])  # actions
      D = tf.placeholder(tf.float32, [nbatch])  # dones
      R = tf.placeholder(tf.float32, [nbatch])  # rewards, not returns
      MU = tf.placeholder(tf.float32, [nbatch, nact])  # mu's
      LR = tf.placeholder(tf.float32, [])
      eps = 1e-6
  
      step_ob_placeholder = tf.placeholder(
          dtype=ob_space.dtype, shape=(nenvs,) + ob_space.shape
      )
      train_ob_placeholder = tf.placeholder(
          dtype=ob_space.dtype,
          shape=(nenvs * (nsteps + 1),) + ob_space.shape,
      )
      with tf.variable_scope("acer_model", reuse=tf.AUTO_REUSE):
          step_model = policy(
              nbatch=nenvs,
              nsteps=1,
              observ_placeholder=step_ob_placeholder,
              sess=sess,
          )
          train_model = policy(
              nbatch=nbatch,
              nsteps=nsteps,
              observ_placeholder=train_ob_placeholder,
              sess=sess,
          )
  
      params = find_trainable_variables("acer_model")
      print("Params {}".format(len(params)))
      for var in params:
          print(var)
  
      # create polyak averaged model
      ema = tf.train.ExponentialMovingAverage(alpha)
      ema_apply_op = ema.apply(params)
  
      def custom_getter(getter, *args, **kwargs):
          v = ema.average(getter(*args, **kwargs))
          print(v.name)
          return v
  
      with tf.variable_scope(
          "acer_model", custom_getter=custom_getter, reuse=True
      ):
          polyak_model = policy(
              nbatch=nbatch,
              nsteps=nsteps,
              observ_placeholder=train_ob_placeholder,
              sess=sess,
          )
  
      # Notation: (var) = batch variable, (var)s = seqeuence variable, (var)_i = variable index by action at step i
  
      # action probability distributions according to train_model, polyak_model and step_model
      # poilcy.pi is probability distribution parameters; to obtain distribution that sums to 1 need to take softmax
      train_model_p = tf.nn.softmax(train_model.pi)
      polyak_model_p = tf.nn.softmax(polyak_model.pi)
      step_model_p = tf.nn.softmax(step_model.pi)
      v = tf.reduce_sum(
          train_model_p * train_model.q, axis=-1
      )  # shape is [nenvs * (nsteps + 1)]
  
      # strip off last step
      f, f_pol, q = map(
          lambda var: strip(var, nenvs, nsteps),
          [train_model_p, polyak_model_p, train_model.q],
      )
      # Get pi and q values for actions taken
      f_i = get_by_index(f, A)
      q_i = get_by_index(q, A)
  
      # Compute ratios for importance truncation
      rho = f / (MU + eps)
      rho_i = get_by_index(rho, A)
  
      # Calculate Q_retrace targets
      qret = q_retrace(R, D, q_i, v, rho_i, nenvs, nsteps, gamma)
  
      # Calculate losses
      # Entropy
      # entropy = tf.reduce_mean(strip(train_model.pd.entropy(), nenvs, nsteps))
      entropy = tf.reduce_mean(cat_entropy_softmax(f))
  
      # Policy Graident loss, with truncated importance sampling & bias correction
      v = strip(v, nenvs, nsteps, True)
      check_shape([qret, v, rho_i, f_i], [[nenvs * nsteps]] * 4)
      check_shape([rho, f, q], [[nenvs * nsteps, nact]] * 2)
  
      # Truncated importance sampling
      adv = qret - v
      logf = tf.log(f_i + eps)
      gain_f = logf * tf.stop_gradient(
          adv * tf.minimum(c, rho_i)
      )  # [nenvs * nsteps]
      loss_f = -tf.reduce_mean(gain_f)
  
      # Bias correction for the truncation
      adv_bc = q - tf.reshape(
          v, [nenvs * nsteps, 1]
      )  # [nenvs * nsteps, nact]
      logf_bc = tf.log(f + eps)  # / (f_old + eps)
      check_shape([adv_bc, logf_bc], [[nenvs * nsteps, nact]] * 2)
      gain_bc = tf.reduce_sum(
          logf_bc
          * tf.stop_gradient(
              adv_bc * tf.nn.relu(1.0 - (c / (rho + eps))) * f
          ),
          axis=1,
      )  # IMP: This is sum, as expectation wrt f
      loss_bc = -tf.reduce_mean(gain_bc)
  
      loss_policy = loss_f + loss_bc
  
      # Value/Q function loss, and explained variance
      check_shape([qret, q_i], [[nenvs * nsteps]] * 2)
      ev = q_explained_variance(
          tf.reshape(q_i, [nenvs, nsteps]),
          tf.reshape(qret, [nenvs, nsteps]),
      )
      loss_q = tf.reduce_mean(
          tf.square(tf.stop_gradient(qret) - q_i) * 0.5
      )
  
      # Net loss
      check_shape([loss_policy, loss_q, entropy], [[]] * 3)
      loss = loss_policy + q_coef * loss_q - ent_coef * entropy
  
      if trust_region:
          g = tf.gradients(
              -(loss_policy - ent_coef * entropy) * nsteps * nenvs, f
          )  # [nenvs * nsteps, nact]
          # k = tf.gradients(KL(f_pol || f), f)
          k = -f_pol / (
              f + eps
          )  # [nenvs * nsteps, nact] # Directly computed gradient of KL divergence wrt f
          k_dot_g = tf.reduce_sum(k * g, axis=-1)
          adj = tf.maximum(
              0.0,
              (tf.reduce_sum(k * g, axis=-1) - delta)
              / (tf.reduce_sum(tf.square(k), axis=-1) + eps),
          )  # [nenvs * nsteps]
  
          # Calculate stats (before doing adjustment) for logging.
          avg_norm_k = avg_norm(k)
          avg_norm_g = avg_norm(g)
          avg_norm_k_dot_g = tf.reduce_mean(tf.abs(k_dot_g))
          avg_norm_adj = tf.reduce_mean(tf.abs(adj))
  
          g = g - tf.reshape(adj, [nenvs * nsteps, 1]) * k
          grads_f = -g / (
              nenvs * nsteps
          )  # These are turst region adjusted gradients wrt f ie statistics of policy pi
          grads_policy = tf.gradients(f, params, grads_f)
          grads_q = tf.gradients(loss_q * q_coef, params)
          grads = [
              gradient_add(g1, g2, param)
              for (g1, g2, param) in zip(grads_policy, grads_q, params)
          ]
  
          avg_norm_grads_f = avg_norm(grads_f) * (nsteps * nenvs)
          norm_grads_q = tf.global_norm(grads_q)
          norm_grads_policy = tf.global_norm(grads_policy)
      else:
          grads = tf.gradients(loss, params)
  
      if max_grad_norm is not None:
          grads, norm_grads = tf.clip_by_global_norm(
              grads, max_grad_norm
          )
      grads = list(zip(grads, params))
      trainer = tf.train.RMSPropOptimizer(
          learning_rate=LR, decay=rprop_alpha, epsilon=rprop_epsilon
      )
      _opt_op = trainer.apply_gradients(grads)
  
      # so when you call _train, you first do the gradient step, then you apply ema
      with tf.control_dependencies([_opt_op]):
          _train = tf.group(ema_apply_op)
  
      lr = Scheduler(v=lr, nvalues=total_timesteps, schedule=lrschedule)
  
      # Ops/Summaries to run, and their names for logging
      run_ops = [
          _train,
          loss,
          loss_q,
          entropy,
          loss_policy,
          loss_f,
          loss_bc,
          ev,
          norm_grads,
      ]
      names_ops = [
          "loss",
          "loss_q",
          "entropy",
          "loss_policy",
          "loss_f",
          "loss_bc",
          "explained_variance",
          "norm_grads",
      ]
      if trust_region:
          run_ops = run_ops + [
              norm_grads_q,
              norm_grads_policy,
              avg_norm_grads_f,
              avg_norm_k,
              avg_norm_g,
              avg_norm_k_dot_g,
              avg_norm_adj,
          ]
          names_ops = names_ops + [
              "norm_grads_q",
              "norm_grads_policy",
              "avg_norm_grads_f",
              "avg_norm_k",
              "avg_norm_g",
              "avg_norm_k_dot_g",
              "avg_norm_adj",
          ]
  
      def train(
          obs, actions, rewards, dones, mus, states, masks, steps
      ):
          cur_lr = lr.value_steps(steps)
          td_map = {
              train_model.X: obs,
              polyak_model.X: obs,
              A: actions,
              R: rewards,
              D: dones,
              MU: mus,
              LR: cur_lr,
          }
          if states is not None:
              td_map[train_model.S] = states
              td_map[train_model.M] = masks
              td_map[polyak_model.S] = states
              td_map[polyak_model.M] = masks
  
          return (
              names_ops,
              sess.run(run_ops, td_map)[1:],
          )  # strip off _train
  
      def _step(observation, **kwargs):
          return step_model._evaluate(
              [step_model.action, step_model_p, step_model.state],
              observation,
              **kwargs,
          )
  
      self.train = train
      self.save = functools.partial(save_variables, sess=sess)
      self.load = functools.partial(load_variables, sess=sess)
      self.train_model = train_model
      self.step_model = step_model
      self._step = _step
      self.step = self.step_model.step
  
      self.initial_state = step_model.initial_state
      tf.global_variables_initializer().run(session=sess)
  
imports: import time
  import functools
  import numpy
  import tensorflow
  from baselines import logger
  from baselines.common import set_global_seeds
  from baselines.common.policies import build_policy
  from baselines.common.tf_util import get_session, save_variables, load_variables
  from baselines.common.vec_env.vec_frame_stack import VecFrameStack
  from baselines.a2c.utils import batch_to_seq, seq_to_batch
  from baselines.a2c.utils import cat_entropy_softmax
  from baselines.a2c.utils import Scheduler, find_trainable_variables
  from baselines.a2c.utils import EpisodeStats
  from baselines.a2c.utils import get_by_index, check_shape, avg_norm, gradient_add, q_explained_variance
  from baselines.acer.buffer import Buffer
  from baselines.acer.runner import Runner
label_desc: Use image node_id 1 to create a new Model object from inherited base classes: object with example: obj = Model(policy, ob_space, ac_space, nenvs, nsteps, ent_coef, q_coef, gamma, max_grad_norm, lr, rprop_alpha, rprop_epsilon, total_timesteps, lrschedule, c, trust_region, alpha, delta)