code: def fill_advantages(self, batch):
      network_keys = self.ap.network_wrappers[
          "main"
      ].input_embedders_parameters.keys()
  
      state_values = []
      for i in range(
          int(batch.size / self.ap.network_wrappers["main"].batch_size)
          + 1
      ):
          start = i * self.ap.network_wrappers["main"].batch_size
          end = (i + 1) * self.ap.network_wrappers["main"].batch_size
          if start == batch.size:
              break
  
          state_values.append(
              self.networks["main"].online_network.predict(
                  {
                      k: v[start:end]
                      for k, v in batch.states(network_keys).items()
                  }
              )[0]
          )
  
      current_state_values = np.concatenate(state_values)
      self.state_values.add_sample(current_state_values)
  
      # calculate advantages
      advantages = []
      value_targets = []
      total_returns = batch.n_step_discounted_rewards()
  
      if (
          self.policy_gradient_rescaler
          == PolicyGradientRescaler.A_VALUE
      ):
          advantages = total_returns - current_state_values
      elif self.policy_gradient_rescaler == PolicyGradientRescaler.GAE:
          # get bootstraps
          episode_start_idx = 0
          advantages = np.array([])
          value_targets = np.array([])
          for idx, game_over in enumerate(batch.game_overs()):
              if game_over:
                  # get advantages for the rollout
                  value_bootstrapping = np.zeros((1,))
                  rollout_state_values = np.append(
                      current_state_values[episode_start_idx : idx + 1],
                      value_bootstrapping,
                  )
  
                  (
                      rollout_advantages,
                      gae_based_value_targets,
                  ) = self.get_general_advantage_estimation_values(
                      batch.rewards()[episode_start_idx : idx + 1],
                      rollout_state_values,
                  )
                  episode_start_idx = idx + 1
                  advantages = np.append(advantages, rollout_advantages)
                  value_targets = np.append(
                      value_targets, gae_based_value_targets
                  )
      else:
          screen.warning(
              "WARNING: The requested policy gradient rescaler is not available"
          )
  
      # standardize
      advantages = (advantages - np.mean(advantages)) / np.std(
          advantages
      )
  
      for transition, advantage, value_target in zip(
          batch.transitions, advantages, value_targets
      ):
          transition.info["advantage"] = advantage
          transition.info["gae_based_value_target"] = value_target
  
      self.action_advantages.add_sample(advantages)
  
imports: import copy
  import math
  from collections import OrderedDict
  from random import shuffle
  from typing import Union
  import numpy
  from rl_coach.agents.actor_critic_agent import ActorCriticAgent
  from rl_coach.agents.policy_optimization_agent import PolicyGradientRescaler
  from rl_coach.architectures.embedder_parameters import InputEmbedderParameters
  from rl_coach.architectures.head_parameters import PPOHeadParameters, VHeadParameters
  from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters
  from rl_coach.base_parameters import AlgorithmParameters, NetworkParameters, AgentParameters
  from rl_coach.core_types import EnvironmentSteps, Batch, EnvResponse, StateType
  from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters
  from rl_coach.exploration_policies.categorical import CategoricalParameters
  from rl_coach.logger import screen
  from rl_coach.memories.episodic.episodic_experience_replay import EpisodicExperienceReplayParameters
  from rl_coach.schedules import ConstantSchedule
  from rl_coach.spaces import DiscreteActionSpace, BoxActionSpace
label_desc: Use image node_id 4 for calling the ClippedPPOAgent obj's underlying member method code with example usage: obj.fill_advantages(batch) without return types