code: def _build_module(self, input_layer):
      if isinstance(self.spaces.action, DiscreteActionSpace):
          self._build_discrete_net(input_layer, self.spaces.action)
      elif isinstance(self.spaces.action, BoxActionSpace):
          self._build_continuous_net(input_layer, self.spaces.action)
      else:
          raise ValueError(
              "only discrete or continuous action spaces are supported for PPO"
          )
  
      self.action_probs_wrt_policy = self.policy_distribution.log_prob(
          self.actions
      )
      self.action_probs_wrt_old_policy = (
          self.old_policy_distribution.log_prob(self.actions)
      )
      self.entropy = tf.reduce_mean(self.policy_distribution.entropy())
  
      # Used by regular PPO only
      # add kl divergence regularization
      self.kl_divergence = tf.reduce_mean(
          tf.distributions.kl_divergence(
              self.old_policy_distribution, self.policy_distribution
          )
      )
  
      if self.use_kl_regularization:
          # no clipping => use kl regularization
          self.weighted_kl_divergence = tf.multiply(
              self.kl_coefficient, self.kl_divergence
          )
          self.regularizations += [
              self.weighted_kl_divergence
              + self.high_kl_penalty_coefficient
              * tf.square(
                  tf.maximum(0.0, self.kl_divergence - self.kl_cutoff)
              )
          ]
  
      # calculate surrogate loss
      self.advantages = tf.placeholder(
          tf.float32, [None], name="advantages"
      )
      self.target = self.advantages
      # action_probs_wrt_old_policy != 0 because it is e^...
      self.likelihood_ratio = tf.exp(
          self.action_probs_wrt_policy
          - self.action_probs_wrt_old_policy
      )
      if self.clip_likelihood_ratio_using_epsilon is not None:
          self.clip_param_rescaler = tf.placeholder(tf.float32, ())
          self.input.append(self.clip_param_rescaler)
          max_value = (
              1
              + self.clip_likelihood_ratio_using_epsilon
              * self.clip_param_rescaler
          )
          min_value = (
              1
              - self.clip_likelihood_ratio_using_epsilon
              * self.clip_param_rescaler
          )
          self.clipped_likelihood_ratio = tf.clip_by_value(
              self.likelihood_ratio, min_value, max_value
          )
          self.scaled_advantages = tf.minimum(
              self.likelihood_ratio * self.advantages,
              self.clipped_likelihood_ratio * self.advantages,
          )
      else:
          self.scaled_advantages = (
              self.likelihood_ratio * self.advantages
          )
      # minus sign is in order to set an objective to minimize (we actually strive for maximizing the surrogate loss)
      self.surrogate_loss = -tf.reduce_mean(self.scaled_advantages)
      if self.is_local:
          # add entropy regularization
          if self.beta:
              self.entropy = tf.reduce_mean(
                  self.policy_distribution.entropy()
              )
              self.regularizations += [
                  -tf.multiply(
                      self.beta,
                      self.entropy,
                      name="entropy_regularization",
                  )
              ]
  
      self.loss = self.surrogate_loss
      tf.losses.add_loss(self.loss)
  
file_path: coach/rl_coach/architectures/tensorflow_components/heads/ppo_head.py
imports: import numpy
  import tensorflow
  from rl_coach.architectures.tensorflow_components.layers import Dense
  from rl_coach.architectures.tensorflow_components.heads.head import Head, normalized_columns_initializer
  from rl_coach.base_parameters import AgentParameters, DistributedTaskParameters
  from rl_coach.core_types import ActionProbabilities
  from rl_coach.spaces import BoxActionSpace, DiscreteActionSpace
  from rl_coach.spaces import SpacesDefinition
  from rl_coach.utils import eps
label_desc: Use image node_id 2 for calling the PPOHead obj's underlying member method code with example usage: obj._build_module(input_layer) without return types