class_docstr: Policy gradient objective, which maximizes the log-likelihood or likelihood-ratio scaled by the
  target reward value (specification key: `policy_gradient`).
  
  Args:
      importance_sampling (bool): Whether to use the importance sampling version of the policy
          gradient objective
          (<span style="color:#00C000"><b>default</b></span>: false).
      clipping_value (parameter, float > 0.0): Clipping threshold for the maximized value
          (<span style="color:#00C000"><b>default</b></span>: no clipping).
      early_reduce (bool): Whether to compute objective for aggregated likelihood instead of
          likelihood per action (<span style="color:#00C000"><b>default</b></span>: true).
      name (string): <span style="color:#0000C0"><b>internal use</b></span>.
      states_spec (specification): <span style="color:#0000C0"><b>internal use</b></span>.
      internals_spec (specification): <span style="color:#0000C0"><b>internal use</b></span>.
      auxiliaries_spec (specification): <span style="color:#0000C0"><b>internal use</b></span>.
      actions_spec (specification): <span style="color:#0000C0"><b>internal use</b></span>.
      reward_spec (specification): <span style="color:#0000C0"><b>internal use</b></span>.
code: def loss(
      self,
      *,
      states,
      horizons,
      internals,
      auxiliaries,
      actions,
      reward,
      reference,
      policy,
      baseline=None,
  ):
      log_probability = self.reference(
          states=states,
          horizons=horizons,
          internals=internals,
          auxiliaries=auxiliaries,
          actions=actions,
          policy=policy,
      )
  
      reference = tf.stop_gradient(input=reference)
  
      if self.importance_sampling:
          log_ratio = log_probability - reference
          # Clip log_ratio for numerical stability (epsilon < 1.0, hence negative)
          log_epsilon = tf_util.constant(
              value=np.log(util.epsilon), dtype="float"
          )
          log_ratio = tf.clip_by_value(
              t=log_ratio,
              clip_value_min=log_epsilon,
              clip_value_max=-log_epsilon,
          )
          target = tf.math.exp(x=log_ratio)
      else:
          target = log_probability
  
      if not self.early_reduce:
          reward = tf.expand_dims(input=reward, axis=1)
  
      if self.clipping_value is None:
          scaled_target = target * reward
  
      else:
          one = tf_util.constant(value=1.0, dtype="float")
          clipping_value = one + self.clipping_value.value()
          if self.importance_sampling:
              min_value = tf.math.reciprocal(x=clipping_value)
              max_value = clipping_value
          else:
              min_value = reference - tf.math.log(x=clipping_value)
              max_value = reference + tf.math.log(x=clipping_value)
  
          clipped_target = tf.clip_by_value(
              t=target,
              clip_value_min=min_value,
              clip_value_max=max_value,
          )
          scaled_target = tf.math.minimum(
              x=(target * reward), y=(clipped_target * reward)
          )
  
      loss = -scaled_target
  
      if not self.early_reduce:
          loss = tf.math.reduce_sum(input_tensor=loss, axis=1)
  
      return loss
  
file_path: tensorforce/tensorforce/core/objectives/policy_gradient.py
imports: import numpy
  import tensorflow
  from tensorforce import util
  from tensorforce.core import parameter_modules, TensorSpec, tf_function, tf_util
  from tensorforce.core.objectives import Objective
label_desc: Use image node_id 5 for calling the PolicyGradient obj's underlying member method code with example usage: obj.loss() and returns: loss