code: import time
  import functools
  import tensorflow as tf
  
  from baselines import logger
  
  from baselines.common import set_global_seeds, explained_variance
  from baselines.common import tf_util
  from baselines.common.policies import build_policy
  
  from baselines.a2c.utils import Scheduler, find_trainable_variables
  from baselines.a2c.runner import Runner
  from baselines.ppo2.ppo2 import safemean
  from collections import deque
  
  from tensorflow import losses
  
  
  class Model(object):
  
      """
      We use this class to :
          __init__:
          - Creates the step_model
          - Creates the train_model
  
          train():
          - Make the training part (feedforward and retropropagation of gradients)
  
          save/load():
          - Save load the model
      """
      def __init__(self, policy, env, nsteps,
                   ent_coef=0.01, vf_coef=0.5, max_grad_norm=0.5, lr=7e-4,
                   alpha=0.99, epsilon=1e-5, total_timesteps=int(80e6), lrschedule='linear', variable_scope="a2c_model"):
          config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
          config.gpu_options.allow_growth = True
          sess = tf_util.get_session(config=config)
          nenvs = env.num_envs
          nbatch = nenvs*nsteps
  
          with tf.variable_scope(variable_scope, reuse=tf.AUTO_REUSE):
              # step_model is used for sampling
              step_model = policy(nenvs, 1, sess)
  
              # train_model is used to train our network
              train_model = policy(nbatch, nsteps, sess)
  
          A = tf.placeholder(train_model.action.dtype, train_model.action.shape)
          ADV = tf.placeholder(tf.float32, [nbatch])
          R = tf.placeholder(tf.float32, [nbatch])
          LR = tf.placeholder(tf.float32, [])
  
          # Calculate the loss
          # Total loss = Policy gradient loss - entropy * entropy coefficient + Value coefficient * value loss
  
          # Policy loss
          neglogpac = train_model.pd.neglogp(A)
          # L = A(s,a) * -logpi(a|s)
          pg_loss = tf.reduce_mean(ADV * neglogpac)
  
          # Entropy is used to improve exploration by limiting the premature convergence to suboptimal policy.
          entropy = tf.reduce_mean(train_model.pd.entropy())
  
          # Value loss
          vf_loss = losses.mean_squared_error(tf.squeeze(train_model.vf), R)
  
          loss = pg_loss - entropy*ent_coef + vf_loss * vf_coef
  
          # Update parameters using loss
          # 1. Get the model parameters
          params = find_trainable_variables(variable_scope)
  
          # 2. Calculate the gradients
          grads = tf.gradients(loss, params)
          if max_grad_norm is not None:
              # Clip the gradients (normalize)
              grads, grad_norm = tf.clip_by_global_norm(grads, max_grad_norm)
          grads = list(zip(grads, params))
          # zip aggregate each gradient with parameters associated
          # For instance zip(ABCD, xyza) => Ax, By, Cz, Da
  
          # 3. Make op for one policy and value update step of A2C
          trainer = tf.train.RMSPropOptimizer(learning_rate=LR, decay=alpha, epsilon=epsilon)
  
          _train = trainer.apply_gradients(grads)
  
          lr = Scheduler(v=lr, nvalues=total_timesteps, schedule=lrschedule)
  
          def train(obs, states, rewards, masks, actions, values):
              # Here we calculate advantage A(s,a) = R + yV(s') - V(s)
              # rewards = R + yV(s')
              advs = rewards - values
              for step in range(len(obs)):
                  cur_lr = lr.value()
  
              td_map = {train_model.X:obs, A:actions, ADV:advs, R:rewards, LR:cur_lr}
              if states is not None:
                  td_map[train_model.S] = states
                  td_map[train_model.M] = masks
              policy_loss, value_loss, policy_entropy, _ = sess.run(
                  [pg_loss, vf_loss, entropy, _train],
                  td_map
              )
              return policy_loss, value_loss, policy_entropy
  
  
          self.train = train
          self.train_model = train_model
          self.step_model = step_model
          self.step = step_model.step
          self.value = step_model.value
          self.initial_state = step_model.initial_state
          self.save = functools.partial(tf_util.save_variables, sess=sess)
          self.load = functools.partial(tf_util.load_variables, sess=sess)
          tf.global_variables_initializer().run(session=sess)
  
  
  def learn(network, env,
            seed=None,
            nsteps=5,
            total_timesteps=int(80e6),
            vf_coef=0.5,
            ent_coef=0.01,
            max_grad_norm=0.5,
            lr=7e-4,
            lrschedule='constant',
            epsilon=1e-5,
            alpha=0.99,
            gamma=0.99,
            log_interval=100,
            load_path=None,
            variable_scope='a2c_model',
            **network_kwargs):
  
      set_global_seeds(seed)
  
      # Get the nb of env
      nenvs = env.num_envs
      policy = build_policy(env, network, **network_kwargs)
  
      # Instantiate the model object (that creates step_model and train_model)
      model = Model(policy=policy, env=env, nsteps=nsteps, ent_coef=ent_coef, vf_coef=vf_coef,
                    max_grad_norm=max_grad_norm, lr=lr, alpha=alpha, epsilon=epsilon, total_timesteps=total_timesteps,
                    lrschedule=lrschedule, variable_scope=variable_scope)
      if load_path is not None:
          model.load(load_path)
  
      # Instantiate the runner object
      runner = Runner(env, model, nsteps=nsteps, gamma=gamma)
      epinfobuf = deque(maxlen=100)
  
      # Calculate the batch_size
      nbatch = nenvs*nsteps
  
      # Start total timer
      tstart = time.time()
  
      for update in range(1, total_timesteps//nbatch+1):
          # Get mini batch of experiences
          obs, states, rewards, masks, actions, values, epinfos = runner.run()
          epinfobuf.extend(epinfos)
  
          policy_loss, value_loss, policy_entropy = model.train(obs, states, rewards, masks, actions, values)
          nseconds = time.time() - tstart
  
          # Calculate the fps (frame per second)
          fps = int((update*nbatch)/nseconds)
          if update % log_interval == 0 or update == 1:
              # Calculates if value function is a good predicator of the returns (ev > 1)
              # or if it's just worse than predicting nothing (ev =< 0)
              ev = explained_variance(values, rewards)
              logger.record_tabular("nupdates", update)
              logger.record_tabular("total_timesteps", update*nbatch)
              logger.record_tabular("fps", fps)
              logger.record_tabular("policy_entropy", float(policy_entropy))
              logger.record_tabular("value_loss", float(value_loss))
              logger.record_tabular("explained_variance", float(ev))
              logger.record_tabular("eprewmean", safemean([epinfo['r'] for epinfo in epinfobuf]))
              logger.record_tabular("eplenmean", safemean([epinfo['l'] for epinfo in epinfobuf]))
              logger.dump_tabular()
      return model
  
apis: tensorflow.train.RMSPropOptimizer tensorflow.reduce_mean tensorflow.gradients tensorflow.placeholder tensorflow.squeeze tensorflow.ConfigProto tensorflow.global_variables_initializer tensorflow.clip_by_global_norm tensorflow.variable_scope
api_extract: [(136, 'baselines.common.set_global_seeds', 'set_global_seeds', (['seed'], {}), False, 'from baselines.common import set_global_seeds, explained_variance\n'), (140, 'baselines.common.policies.build_policy', 'build_policy', (['env', 'network'], {}), False, 'from baselines.common.policies import build_policy\n'), (150, 'baselines.a2c.runner.Runner', 'Runner', (['env', 'model'], {'nsteps': 'nsteps', 'gamma': 'gamma'}), False, 'from baselines.a2c.runner import Runner\n'), (151, 'collections.deque', 'deque', ([], {'maxlen': '(100)'}), False, 'from collections import deque\n'), (157, 'time.time', 'time.time', ([], {}), False, 'import time\n'), (36, 'tensorflow.ConfigProto', 'tf.ConfigProto', ([], {'log_device_placement': '(False)', 'allow_soft_placement': '(True)'}), True, 'import tensorflow as tf\n'), (38, 'baselines.common.tf_util.get_session', 'tf_util.get_session', ([], {'config': 'config'}), False, 'from baselines.common import tf_util\n'), (49, 'tensorflow.placeholder', 'tf.placeholder', (['train_model.action.dtype', 'train_model.action.shape'], {}), True, 'import tensorflow as tf\n'), (50, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[nbatch]'], {}), True, 'import tensorflow as tf\n'), (51, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[nbatch]'], {}), True, 'import tensorflow as tf\n'), (52, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[]'], {}), True, 'import tensorflow as tf\n'), (60, 'tensorflow.reduce_mean', 'tf.reduce_mean', (['(ADV * neglogpac)'], {}), True, 'import tensorflow as tf\n'), (72, 'baselines.a2c.utils.find_trainable_variables', 'find_trainable_variables', (['variable_scope'], {}), False, 'from baselines.a2c.utils import Scheduler, find_trainable_variables\n'), (75, 'tensorflow.gradients', 'tf.gradients', (['loss', 'params'], {}), True, 'import tensorflow as tf\n'), (84, 'tensorflow.train.RMSPropOptimizer', 'tf.train.RMSPropOptimizer', ([], {'learning_rate': 'LR', 'decay': 'alpha', 'epsilon': 'epsilon'}), True, 'import tensorflow as tf\n'), (88, 'baselines.a2c.utils.Scheduler', 'Scheduler', ([], {'v': 'lr', 'nvalues': 'total_timesteps', 'schedule': 'lrschedule'}), False, 'from baselines.a2c.utils import Scheduler, find_trainable_variables\n'), (114, 'functools.partial', 'functools.partial', (['tf_util.save_variables'], {'sess': 'sess'}), False, 'import functools\n'), (115, 'functools.partial', 'functools.partial', (['tf_util.load_variables'], {'sess': 'sess'}), False, 'import functools\n'), (42, 'tensorflow.variable_scope', 'tf.variable_scope', (['variable_scope'], {'reuse': 'tf.AUTO_REUSE'}), True, 'import tensorflow as tf\n'), (66, 'tensorflow.squeeze', 'tf.squeeze', (['train_model.vf'], {}), True, 'import tensorflow as tf\n'), (78, 'tensorflow.clip_by_global_norm', 'tf.clip_by_global_norm', (['grads', 'max_grad_norm'], {}), True, 'import tensorflow as tf\n'), (165, 'time.time', 'time.time', ([], {}), False, 'import time\n'), (172, 'baselines.common.explained_variance', 'explained_variance', (['values', 'rewards'], {}), False, 'from baselines.common import set_global_seeds, explained_variance\n'), (173, 'baselines.logger.record_tabular', 'logger.record_tabular', (['"""nupdates"""', 'update'], {}), False, 'from baselines import logger\n'), (174, 'baselines.logger.record_tabular', 'logger.record_tabular', (['"""total_timesteps"""', '(update * nbatch)'], {}), False, 'from baselines import logger\n'), (175, 'baselines.logger.record_tabular', 'logger.record_tabular', (['"""fps"""', 'fps'], {}), False, 'from baselines import logger\n'), (181, 'baselines.logger.dump_tabular', 'logger.dump_tabular', ([], {}), False, 'from baselines import logger\n'), (116, 'tensorflow.global_variables_initializer', 'tf.global_variables_initializer', ([], {}), True, 'import tensorflow as tf\n'), (179, 'baselines.ppo2.ppo2.safemean', 'safemean', (["[epinfo['r'] for epinfo in epinfobuf]"], {}), False, 'from baselines.ppo2.ppo2 import safemean\n'), (180, 'baselines.ppo2.ppo2.safemean', 'safemean', (["[epinfo['l'] for epinfo in epinfobuf]"], {}), False, 'from baselines.ppo2.ppo2 import safemean\n')]