docstr: Used for forward pass through loss computations.
  Calculates two losses (L2 and a clipped difference L2 loss) and takes the maximum of the two.
  Works with batches of data, and optionally time_steps, but be consistent in usage: i.e. if using time_step,
  new_policy_values, old_policy_values and target_values all must include a time_step dimension.
  
  :param (mx.nd or mx.sym) F: backend api (mx.sym if block has been hybridized).
  :param new_policy_values: values predicted by PPOVHead network,
      of shape (batch_size) or
      of shape (batch_size, time_step).
  :param old_policy_values: values predicted by old value network,
      of shape (batch_size) or
      of shape (batch_size, time_step).
  :param target_values: actual state values,
      of shape (batch_size) or
      of shape (batch_size, time_step).
  :return: loss, of shape (batch_size).
docstr_tok: ["Used","for","forward","pass","through","loss","computations",".","Calculates","two","losses","(","L2","and","a","clipped","difference","L2","loss",")","and","takes","the","maximum","of","the","two",".","Works","with","batches","of","data",",","and","optionally","time_steps",",","but","be","consistent","in","usage",":","i.e",".","if","using","time_step",",","new_policy_values",",","old_policy_values","and","target_values","all","must","include","a","time_step","dimension",".",":","param","(","mx.nd","or","mx.sym",")","F",":","backend","api","(","mx.sym","if","block","has","been","hybridized",")",".",":","param","new_policy_values",":","values","predicted","by","PPOVHead","network",",","of","shape","(","batch_size",")","or","of","shape","(","batch_size",",","time_step",")",".",":","param","old_policy_values",":","values","predicted","by","old","value","network",",","of","shape","(","batch_size",")","or","of","shape","(","batch_size",",","time_step",")",".",":","param","target_values",":","actual","state","values",",","of","shape","(","batch_size",")","or","of","shape","(","batch_size",",","time_step",")",".",":","return",":","loss",",","of","shape","(","batch_size",")","."]
code: def loss_forward(
      self,
      F: ModuleType,
      new_policy_values: nd_sym_type,
      old_policy_values: nd_sym_type,
      target_values: nd_sym_type,
  ) -> List[Tuple[nd_sym_type, str]]:
      """
      Used for forward pass through loss computations.
      Calculates two losses (L2 and a clipped difference L2 loss) and takes the maximum of the two.
      Works with batches of data, and optionally time_steps, but be consistent in usage: i.e. if using time_step,
      new_policy_values, old_policy_values and target_values all must include a time_step dimension.
  
      :param (mx.nd or mx.sym) F: backend api (mx.sym if block has been hybridized).
      :param new_policy_values: values predicted by PPOVHead network,
          of shape (batch_size) or
          of shape (batch_size, time_step).
      :param old_policy_values: values predicted by old value network,
          of shape (batch_size) or
          of shape (batch_size, time_step).
      :param target_values: actual state values,
          of shape (batch_size) or
          of shape (batch_size, time_step).
      :return: loss, of shape (batch_size).
      """
      # L2 loss
      value_loss_1 = (new_policy_values - target_values).square()
      # Clipped difference L2 loss
      diff = new_policy_values - old_policy_values
      clipped_diff = diff.clip(
          a_min=-self.clip_likelihood_ratio_using_epsilon,
          a_max=self.clip_likelihood_ratio_using_epsilon,
      )
      value_loss_2 = (
          old_policy_values + clipped_diff - target_values
      ).square()
      # Maximum of the two losses, element-wise maximum.
      value_loss_max = mx.nd.stack(value_loss_1, value_loss_2).max(
          axis=0
      )
      # Aggregate over temporal axis, adding if doesn't exist (hense reshape)
      value_loss_max_w_time = value_loss_max.reshape(shape=(0, -1))
      value_loss = value_loss_max_w_time.mean(axis=1)
      # Weight the loss (and average over samples of batch)
      value_loss_weighted = value_loss.mean() * self.weight
      return [(value_loss_weighted, LOSS_OUT_TYPE_LOSS)]
  
file_path: coach/rl_coach/architectures/mxnet_components/heads/ppo_v_head.py
imports: from typing import List, Tuple, Union
  from types import ModuleType
  import mxnet
  from mxnet.gluon import nn
  from rl_coach.architectures.mxnet_components.heads.head import Head, HeadLoss, LossInputSchema, NormalizedRSSInitializer
  from rl_coach.architectures.mxnet_components.heads.head import LOSS_OUT_TYPE_LOSS
  from rl_coach.base_parameters import AgentParameters
  from rl_coach.core_types import ActionProbabilities
  from rl_coach.spaces import SpacesDefinition
label_desc: Use image node_id 3 for calling the PPOVHeadLoss obj's underlying member method code with example usage: obj.loss_forward(F, new_policy_values, old_policy_values, target_values) and returns: list