code: def vpg(
      env_fn,
      actor_critic=core.mlp_actor_critic,
      ac_kwargs=dict(),
      seed=0,
      steps_per_epoch=4000,
      epochs=50,
      gamma=0.99,
      pi_lr=3e-4,
      vf_lr=1e-3,
      train_v_iters=80,
      lam=0.97,
      max_ep_len=1000,
      logger_kwargs=dict(),
      save_freq=10,
  ):
      """
      Vanilla Policy Gradient
  
      (with GAE-Lambda for advantage estimation)
  
      Args:
          env_fn : A function which creates a copy of the environment.
              The environment must satisfy the OpenAI Gym API.
  
          actor_critic: A function which takes in placeholder symbols
              for state, ``x_ph``, and action, ``a_ph``, and returns the main
              outputs from the agent's Tensorflow computation graph:
  
              ===========  ================  ======================================
              Symbol       Shape             Description
              ===========  ================  ======================================
              ``pi``       (batch, act_dim)  | Samples actions from policy given
                                             | states.
              ``logp``     (batch,)          | Gives log probability, according to
                                             | the policy, of taking actions ``a_ph``
                                             | in states ``x_ph``.
              ``logp_pi``  (batch,)          | Gives log probability, according to
                                             | the policy, of the action sampled by
                                             | ``pi``.
              ``v``        (batch,)          | Gives the value estimate for states
                                             | in ``x_ph``. (Critical: make sure
                                             | to flatten this!)
              ===========  ================  ======================================
  
          ac_kwargs (dict): Any kwargs appropriate for the actor_critic
              function you provided to VPG.
  
          seed (int): Seed for random number generators.
  
          steps_per_epoch (int): Number of steps of interaction (state-action pairs)
              for the agent and the environment in each epoch.
  
          epochs (int): Number of epochs of interaction (equivalent to
              number of policy updates) to perform.
  
          gamma (float): Discount factor. (Always between 0 and 1.)
  
          pi_lr (float): Learning rate for policy optimizer.
  
          vf_lr (float): Learning rate for value function optimizer.
  
          train_v_iters (int): Number of gradient descent steps to take on
              value function per epoch.
  
          lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
              close to 1.)
  
          max_ep_len (int): Maximum length of trajectory / episode / rollout.
  
          logger_kwargs (dict): Keyword args for EpochLogger.
  
          save_freq (int): How often (in terms of gap between epochs) to save
              the current policy and value function.
  
      """
  
      logger = EpochLogger(**logger_kwargs)
      logger.save_config(locals())
  
      seed += 10000 * proc_id()
      tf.set_random_seed(seed)
      np.random.seed(seed)
  
      env = env_fn()
      obs_dim = env.observation_space.shape
      act_dim = env.action_space.shape
  
      # Share information about action space with policy architecture
      ac_kwargs["action_space"] = env.action_space
  
      # Inputs to computation graph
      x_ph, a_ph = core.placeholders_from_spaces(
          env.observation_space, env.action_space
      )
      adv_ph, ret_ph, logp_old_ph = core.placeholders(None, None, None)
  
      # Main outputs from computation graph
      pi, logp, logp_pi, v = actor_critic(x_ph, a_ph, **ac_kwargs)
  
      # Need all placeholders in *this* order later (to zip with data from buffer)
      all_phs = [x_ph, a_ph, adv_ph, ret_ph, logp_old_ph]
  
      # Every step, get: action, value, and logprob
      get_action_ops = [pi, v, logp_pi]
  
      # Experience buffer
      local_steps_per_epoch = int(steps_per_epoch / num_procs())
      buf = VPGBuffer(
          obs_dim, act_dim, local_steps_per_epoch, gamma, lam
      )
  
      # Count variables
      var_counts = tuple(
          core.count_vars(scope) for scope in ["pi", "v"]
      )
      logger.log(
          "\nNumber of parameters: \t pi: %d, \t v: %d\n" % var_counts
      )
  
      # VPG objectives
      pi_loss = -tf.reduce_mean(logp * adv_ph)
      v_loss = tf.reduce_mean((ret_ph - v) ** 2)
  
      # Info (useful to watch during learning)
      approx_kl = tf.reduce_mean(
          logp_old_ph - logp
      )  # a sample estimate for KL-divergence, easy to compute
      approx_ent = tf.reduce_mean(
          -logp
      )  # a sample estimate for entropy, also easy to compute
  
      # Optimizers
      train_pi = MpiAdamOptimizer(learning_rate=pi_lr).minimize(pi_loss)
      train_v = MpiAdamOptimizer(learning_rate=vf_lr).minimize(v_loss)
  
      sess = tf.Session()
      sess.run(tf.global_variables_initializer())
  
      # Sync params across processes
      sess.run(sync_all_params())
  
      # Setup model saving
      logger.setup_tf_saver(
          sess, inputs={"x": x_ph}, outputs={"pi": pi, "v": v}
      )
  
      def update():
          inputs = {k: v for k, v in zip(all_phs, buf.get())}
          pi_l_old, v_l_old, ent = sess.run(
              [pi_loss, v_loss, approx_ent], feed_dict=inputs
          )
  
          # Policy gradient step
          sess.run(train_pi, feed_dict=inputs)
  
          # Value function learning
          for _ in range(train_v_iters):
              sess.run(train_v, feed_dict=inputs)
  
          # Log changes from update
          pi_l_new, v_l_new, kl = sess.run(
              [pi_loss, v_loss, approx_kl], feed_dict=inputs
          )
          logger.store(
              LossPi=pi_l_old,
              LossV=v_l_old,
              KL=kl,
              Entropy=ent,
              DeltaLossPi=(pi_l_new - pi_l_old),
              DeltaLossV=(v_l_new - v_l_old),
          )
  
      start_time = time.time()
      o, ep_ret, ep_len = env.reset(), 0, 0
  
      # Main loop: collect experience in env and update/log each epoch
      for epoch in range(epochs):
          for t in range(local_steps_per_epoch):
              a, v_t, logp_t = sess.run(
                  get_action_ops, feed_dict={x_ph: o.reshape(1, -1)}
              )
  
              o2, r, d, _ = env.step(a[0])
              ep_ret += r
              ep_len += 1
  
              # save and log
              buf.store(o, a, r, v_t, logp_t)
              logger.store(VVals=v_t)
  
              # Update obs (critical!)
              o = o2
  
              terminal = d or (ep_len == max_ep_len)
              if terminal or (t == local_steps_per_epoch - 1):
                  if not (terminal):
                      print(
                          "Warning: trajectory cut off by epoch at %d steps."
                          % ep_len
                      )
                  # if trajectory didn't reach terminal state, bootstrap value target
                  last_val = (
                      0
                      if d
                      else sess.run(
                          v, feed_dict={x_ph: o.reshape(1, -1)}
                      )
                  )
                  buf.finish_path(last_val)
                  if terminal:
                      # only save EpRet / EpLen if trajectory finished
                      logger.store(EpRet=ep_ret, EpLen=ep_len)
                  o, ep_ret, ep_len = env.reset(), 0, 0
  
          # Save model
          if (epoch % save_freq == 0) or (epoch == epochs - 1):
              logger.save_state({"env": env}, None)
  
          # Perform VPG update!
          update()
  
          # Log info about epoch
          logger.log_tabular("Epoch", epoch)
          logger.log_tabular("EpRet", with_min_and_max=True)
          logger.log_tabular("EpLen", average_only=True)
          logger.log_tabular("VVals", with_min_and_max=True)
          logger.log_tabular(
              "TotalEnvInteracts", (epoch + 1) * steps_per_epoch
          )
          logger.log_tabular("LossPi", average_only=True)
          logger.log_tabular("LossV", average_only=True)
          logger.log_tabular("DeltaLossPi", average_only=True)
          logger.log_tabular("DeltaLossV", average_only=True)
          logger.log_tabular("Entropy", average_only=True)
          logger.log_tabular("KL", average_only=True)
          logger.log_tabular("Time", time.time() - start_time)
          logger.dump_tabular()
  
file_path: spinningup/spinup/algos/tf1/vpg/vpg.py
imports: import numpy
  import tensorflow
  import gym
  import time
  import spinup.algos.tf1.vpg.core
  from spinup.utils.logx import EpochLogger
  from spinup.utils.mpi_tf import MpiAdamOptimizer, sync_all_params
  from spinup.utils.mpi_tools import mpi_fork, mpi_avg, proc_id, mpi_statistics_scalar, num_procs
label_desc: Use image node_id 1 for calling a global function with example usage: vpg(env_fn, actor_critic, ac_kwargs, seed, steps_per_epoch, epochs, gamma, pi_lr, vf_lr, train_v_iters, lam, max_ep_len, logger_kwargs, save_freq) without return types