class_docstr: A PPO Agent.
class_docstr_tok: ["A","PPO","Agent","."]
docstr: Computes the value estimation loss for actor-critic training.
  
  All tensors should have a single batch dimension.
  
  Args:
    time_steps: A batch of timesteps.
    returns: Per-timestep returns for value function to predict. (Should come
      from TD-lambda computation.)
    weights: Optional scalar or element-wise (per-batch-entry) importance
      weights.  Includes a mask for invalid timesteps.
    old_value_predictions: (Optional) The saved value predictions from
      policy_info, required when self._value_clipping > 0.
    debug_summaries: True if debug summaries should be created.
    training: Whether this loss is going to be used for training.
  
  Returns:
    value_estimation_loss: A scalar value_estimation_loss loss.
  
  Raises:
    ValueError: If old_value_predictions was not passed in, but value clipping
      was performed.
docstr_tok: ["Computes","the","value","estimation","loss","for","actor-critic","training",".","All","tensors","should","have","a","single","batch","dimension",".","Args",":","time_steps",":","A","batch","of","timesteps",".","returns",":","Per-timestep","returns","for","value","function","to","predict",".","(","Should","come","from","TD-lambda","computation",".",")","weights",":","Optional","scalar","or","element-wise","(","per-batch-entry",")","importance","weights",".","Includes","a","mask","for","invalid","timesteps",".","old_value_predictions",":","(","Optional",")","The","saved","value","predictions","from","policy_info",",","required","when","self._value_clipping",">","0.","debug_summaries",":","True","if","debug","summaries","should","be","created",".","training",":","Whether","this","loss","is","going","to","be","used","for","training",".","Returns",":","value_estimation_loss",":","A","scalar","value_estimation_loss","loss",".","Raises",":","ValueError",":","If","old_value_predictions","was","not","passed","in",",","but","value","clipping","was","performed","."]
code: def value_estimation_loss(
      self,
      time_steps: ts.TimeStep,
      returns: types.Tensor,
      weights: types.Tensor,
      old_value_predictions: Optional[types.Tensor] = None,
      debug_summaries: bool = False,
      training: bool = False,
  ) -> types.Tensor:
      """Computes the value estimation loss for actor-critic training.
  
      All tensors should have a single batch dimension.
  
      Args:
        time_steps: A batch of timesteps.
        returns: Per-timestep returns for value function to predict. (Should come
          from TD-lambda computation.)
        weights: Optional scalar or element-wise (per-batch-entry) importance
          weights.  Includes a mask for invalid timesteps.
        old_value_predictions: (Optional) The saved value predictions from
          policy_info, required when self._value_clipping > 0.
        debug_summaries: True if debug summaries should be created.
        training: Whether this loss is going to be used for training.
  
      Returns:
        value_estimation_loss: A scalar value_estimation_loss loss.
  
      Raises:
        ValueError: If old_value_predictions was not passed in, but value clipping
          was performed.
      """
  
      observation = time_steps.observation
      # TODO(b/171573175): remove the condition once histograms are
      # supported on TPUs.
      if debug_summaries and not tf.config.list_logical_devices("TPU"):
          observation_list = tf.nest.flatten(observation)
          show_observation_index = len(observation_list) != 1
          for i, single_observation in enumerate(observation_list):
              observation_name = (
                  "observations_{}".format(i)
                  if show_observation_index
                  else "observations"
              )
              tf.compat.v2.summary.histogram(
                  name=observation_name,
                  data=single_observation,
                  step=self.train_step_counter,
              )
  
      batch_size = nest_utils.get_outer_shape(
          time_steps, self._time_step_spec
      )[0]
      value_state = self._collect_policy.get_initial_value_state(
          batch_size
      )
  
      value_preds, _ = self._collect_policy.apply_value_network(
          time_steps.observation,
          time_steps.step_type,
          value_state=value_state,
          training=training,
      )
      value_estimation_error = tf.math.squared_difference(
          returns, value_preds
      )
  
      if self._value_clipping > 0:
          if old_value_predictions is None:
              raise ValueError(
                  "old_value_predictions is None but needed for value clipping."
              )
          clipped_value_preds = (
              old_value_predictions
              + tf.clip_by_value(
                  value_preds - old_value_predictions,
                  -self._value_clipping,
                  self._value_clipping,
              )
          )
          clipped_value_estimation_error = tf.math.squared_difference(
              returns, clipped_value_preds
          )
          value_estimation_error = tf.maximum(
              value_estimation_error, clipped_value_estimation_error
          )
  
      if self._aggregate_losses_across_replicas:
          value_estimation_loss = (
              common.aggregate_losses(
                  per_example_loss=value_estimation_error,
                  sample_weight=weights,
              ).total_loss
              * self._value_pred_loss_coef
          )
      else:
          value_estimation_loss = (
              tf.math.reduce_mean(value_estimation_error * weights)
              * self._value_pred_loss_coef
          )
  
      if debug_summaries:
          tf.compat.v2.summary.scalar(
              name="value_pred_avg",
              data=tf.reduce_mean(input_tensor=value_preds),
              step=self.train_step_counter,
          )
          tf.compat.v2.summary.scalar(
              name="value_actual_avg",
              data=tf.reduce_mean(input_tensor=returns),
              step=self.train_step_counter,
          )
          tf.compat.v2.summary.scalar(
              name="value_estimation_loss",
              data=value_estimation_loss,
              step=self.train_step_counter,
          )
          # TODO(b/171573175): remove the condition once histograms are supported
          # on TPUs.
          if not tf.config.list_logical_devices("TPU"):
              tf.compat.v2.summary.histogram(
                  name="value_preds",
                  data=value_preds,
                  step=self.train_step_counter,
              )
              tf.compat.v2.summary.histogram(
                  name="value_estimation_error",
                  data=value_estimation_error,
                  step=self.train_step_counter,
              )
  
      if self._check_numerics:
          value_estimation_loss = tf.debugging.check_numerics(
              value_estimation_loss, "value_estimation_loss"
          )
  
      return value_estimation_loss
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import collections
  from typing import Optional, Text, Tuple
  from absl import logging
  import gin
  from six.moves import range
  from six.moves import zip
  import tensorflow
  from tf_agents.agents import data_converter
  from tf_agents.agents import tf_agent
  from tf_agents.agents.ppo import ppo_policy
  from tf_agents.agents.ppo import ppo_utils
  from tf_agents.networks import network
  from tf_agents.policies import greedy_policy
  from tf_agents.specs import tensor_spec
  from tf_agents.trajectories import time_step
  from tf_agents.trajectories import trajectory
  from tf_agents.typing import types
  from tf_agents.utils import common
  from tf_agents.utils import eager_utils
  from tf_agents.utils import nest_utils
  from tf_agents.utils import object_identity
  from tf_agents.utils import tensor_normalizer
  from tf_agents.utils import value_ops
label_desc: Use image node_id 14 for calling the PPOAgent obj's underlying member method code with example usage: obj.value_estimation_loss(time_steps, returns, weights, old_value_predictions, debug_summaries, training) and returns: value_estimation_loss