code: def _build_module(self, input_layer):
      if isinstance(self.spaces.action, DiscreteActionSpace):
          # create a discrete action network (softmax probabilities output)
          self._build_discrete_net(input_layer, self.spaces.action)
      else:
          raise ValueError(
              "only discrete action spaces are supported for ACER"
          )
  
      if self.is_local:
          # add entropy regularization
          if self.beta:
              self.entropy = tf.reduce_mean(
                  self.policy_distribution.entropy()
              )
              self.regularizations += [
                  -tf.multiply(
                      self.beta,
                      self.entropy,
                      name="entropy_regularization",
                  )
              ]
  
          # Truncated importance sampling with bias corrections
          importance_sampling_weight = tf.placeholder(
              tf.float32,
              [None, self.num_actions],
              name="{}_importance_sampling_weight".format(
                  self.get_name()
              ),
          )
          self.input.append(importance_sampling_weight)
          importance_sampling_weight_i = tf.placeholder(
              tf.float32,
              [None],
              name="{}_importance_sampling_weight_i".format(
                  self.get_name()
              ),
          )
          self.input.append(importance_sampling_weight_i)
  
          V_values = tf.placeholder(
              tf.float32,
              [None],
              name="{}_V_values".format(self.get_name()),
          )
          self.target.append(V_values)
          Q_values = tf.placeholder(
              tf.float32,
              [None, self.num_actions],
              name="{}_Q_values".format(self.get_name()),
          )
          self.input.append(Q_values)
          Q_retrace = tf.placeholder(
              tf.float32,
              [None],
              name="{}_Q_retrace".format(self.get_name()),
          )
          self.input.append(Q_retrace)
  
          action_log_probs_wrt_policy = (
              self.policy_distribution.log_prob(self.actions)
          )
          self.probability_loss = -tf.reduce_mean(
              action_log_probs_wrt_policy
              * (Q_retrace - V_values)
              * tf.minimum(
                  self.ap.algorithm.importance_weight_truncation,
                  importance_sampling_weight_i,
              )
          )
  
          log_probs_wrt_policy = tf.log(self.policy_probs + eps)
          bias_correction_gain = tf.reduce_sum(
              log_probs_wrt_policy
              * (Q_values - tf.expand_dims(V_values, 1))
              * tf.nn.relu(
                  1.0
                  - (
                      self.ap.algorithm.importance_weight_truncation
                      / (importance_sampling_weight + eps)
                  )
              )
              * tf.stop_gradient(self.policy_probs),
              axis=1,
          )
          self.bias_correction_loss = -tf.reduce_mean(
              bias_correction_gain
          )
  
          self.loss = self.probability_loss + self.bias_correction_loss
          tf.losses.add_loss(self.loss)
  
          # Trust region
          batch_size = tf.to_float(tf.shape(input_layer)[0])
          average_policy = tf.placeholder(
              tf.float32,
              [None, self.num_actions],
              name="{}_average_policy".format(self.get_name()),
          )
          self.input.append(average_policy)
          average_policy_distribution = (
              tf.contrib.distributions.Categorical(
                  probs=(average_policy + eps)
              )
          )
          self.kl_divergence = tf.reduce_mean(
              tf.distributions.kl_divergence(
                  average_policy_distribution, self.policy_distribution
              )
          )
          if self.ap.algorithm.use_trust_region_optimization:
  
              @tf.custom_gradient
              def trust_region_layer(x):
                  def grad(g):
                      g = -g * batch_size
                      k = -average_policy / (self.policy_probs + eps)
                      adj = tf.nn.relu(
                          (
                              tf.reduce_sum(k * g, axis=1)
                              - self.ap.algorithm.max_KL_divergence
                          )
                          / (tf.reduce_sum(tf.square(k), axis=1) + eps)
                      )
                      g = g - tf.expand_dims(adj, 1) * k
                      return -g / batch_size
  
                  return tf.identity(x), grad
  
              self.output = trust_region_layer(self.output)
  
file_path: coach/rl_coach/architectures/tensorflow_components/heads/acer_policy_head.py
imports: import tensorflow
  from rl_coach.architectures.tensorflow_components.layers import Dense
  from rl_coach.architectures.tensorflow_components.heads.head import Head
  from rl_coach.base_parameters import AgentParameters
  from rl_coach.core_types import ActionProbabilities
  from rl_coach.spaces import DiscreteActionSpace
  from rl_coach.spaces import SpacesDefinition
  from rl_coach.utils import eps
label_desc: Use image node_id 2 for calling the ACERPolicyHead obj's underlying member method code with example usage: obj._build_module(input_layer) and returns: tf,  grad, unknown