docstr: Head block for Proximal Policy Optimization, to calculate probabilities for each action given middleware
  representation of the environment state.
  
  :param agent_parameters: containing algorithm parameters such as clip_likelihood_ratio_using_epsilon
      and beta_entropy.
  :param spaces: containing action spaces used for defining size of network output.
  :param network_name: name of head network. currently unused.
  :param head_type_idx: index of head network. currently unused.
  :param loss_weight: scalar used to adjust relative weight of loss (if using this loss with others).
  :param is_local: flag to denote if network is local. currently unused.
  :param activation_function: activation function to use between layers. currently unused.
  :param dense_layer: type of dense layer to use in network. currently unused.
docstr_tok: ["Head","block","for","Proximal","Policy","Optimization",",","to","calculate","probabilities","for","each","action","given","middleware","representation","of","the","environment","state",".",":","param","agent_parameters",":","containing","algorithm","parameters","such","as","clip_likelihood_ratio_using_epsilon","and","beta_entropy",".",":","param","spaces",":","containing","action","spaces","used","for","defining","size","of","network","output",".",":","param","network_name",":","name","of","head","network",".","currently","unused",".",":","param","head_type_idx",":","index","of","head","network",".","currently","unused",".",":","param","loss_weight",":","scalar","used","to","adjust","relative","weight","of","loss","(","if","using","this","loss","with","others",")",".",":","param","is_local",":","flag","to","denote","if","network","is","local",".","currently","unused",".",":","param","activation_function",":","activation","function","to","use","between","layers",".","currently","unused",".",":","param","dense_layer",":","type","of","dense","layer","to","use","in","network",".","currently","unused","."]
code: def __init__(
      self,
      agent_parameters: AgentParameters,
      spaces: SpacesDefinition,
      network_name: str,
      head_type_idx: int = 0,
      loss_weight: float = 1.0,
      is_local: bool = True,
      activation_function: str = "tanh",
      dense_layer: None = None,
  ) -> None:
      """
      Head block for Proximal Policy Optimization, to calculate probabilities for each action given middleware
      representation of the environment state.
  
      :param agent_parameters: containing algorithm parameters such as clip_likelihood_ratio_using_epsilon
          and beta_entropy.
      :param spaces: containing action spaces used for defining size of network output.
      :param network_name: name of head network. currently unused.
      :param head_type_idx: index of head network. currently unused.
      :param loss_weight: scalar used to adjust relative weight of loss (if using this loss with others).
      :param is_local: flag to denote if network is local. currently unused.
      :param activation_function: activation function to use between layers. currently unused.
      :param dense_layer: type of dense layer to use in network. currently unused.
      """
      super().__init__(
          agent_parameters,
          spaces,
          network_name,
          head_type_idx,
          loss_weight,
          is_local,
          activation_function,
          dense_layer=dense_layer,
      )
      self.return_type = ActionProbabilities
  
      self.clip_likelihood_ratio_using_epsilon = (
          agent_parameters.algorithm.clip_likelihood_ratio_using_epsilon
      )
      self.beta = agent_parameters.algorithm.beta_entropy
      self.use_kl_regularization = (
          agent_parameters.algorithm.use_kl_regularization
      )
      if self.use_kl_regularization:
          self.initial_kl_coefficient = (
              agent_parameters.algorithm.initial_kl_coefficient
          )
          self.kl_cutoff = (
              2 * agent_parameters.algorithm.target_kl_divergence
          )
          self.high_kl_penalty_coefficient = (
              agent_parameters.algorithm.high_kl_penalty_coefficient
          )
      else:
          (
              self.initial_kl_coefficient,
              self.kl_cutoff,
              self.high_kl_penalty_coefficient,
          ) = (None, None, None)
      self._loss = []
  
      if isinstance(self.spaces.action, DiscreteActionSpace):
          self.net = DiscretePPOHead(
              num_actions=len(self.spaces.action.actions)
          )
      elif isinstance(self.spaces.action, BoxActionSpace):
          self.net = ContinuousPPOHead(
              num_actions=self.spaces.action.shape[0]
          )
      else:
          raise ValueError(
              "Only discrete or continuous action spaces are supported for PPO."
          )
  
imports: from typing import List, Tuple, Union
  from types import ModuleType
  import math
  import mxnet
  from mxnet.gluon import nn
  from rl_coach.base_parameters import AgentParameters
  from rl_coach.core_types import ActionProbabilities
  from rl_coach.spaces import SpacesDefinition, BoxActionSpace, DiscreteActionSpace
  from rl_coach.utils import eps
  from rl_coach.architectures.mxnet_components.heads.head import Head, HeadLoss, LossInputSchema, NormalizedRSSInitializer
  from rl_coach.architectures.mxnet_components.heads.head import LOSS_OUT_TYPE_LOSS, LOSS_OUT_TYPE_REGULARIZATION
  from rl_coach.architectures.mxnet_components.utils import hybrid_clip, broadcast_like
label_desc: Use image node_id 1 to create a new PPOHead object from inherited base classes: Head with example: obj = PPOHead(agent_parameters, spaces, network_name, head_type_idx, loss_weight, is_local, activation_function, dense_layer)