class_docstr: Implements DreamerV3 losses and gradient-based update logic in TensorFlow.
  
  The critic EMA-copy update step can be found in the `DreamerV3Learner` base class,
  as it is framework independent.
  
  We define 3 local TensorFlow optimizers for the sub components "world_model",
  "actor", and "critic". Each of these optimizers might use a different learning rate,
  epsilon parameter, and gradient clipping thresholds and procedures.
docstr: Helper method computing the actor's loss terms.
  
  Args:
      module_id: The module_id for which to compute the actor loss.
      config: The DreamerV3Config to use.
      dream_data: The data generated by dreaming for H steps (horizon) starting
          from any BxT state (sampled from the buffer for the train batch).
      value_targets_t0_to_Hm1_BxT: The computed value function targets of the
          shape (t0 to H-1, BxT).
  
  Returns:
      The total actor loss tensor.
docstr_tok: ["Helper","method","computing","the","actor","'s","loss","terms",".","Args",":","module_id",":","The","module_id","for","which","to","compute","the","actor","loss",".","config",":","The","DreamerV3Config","to","use",".","dream_data",":","The","data","generated","by","dreaming","for","H","steps","(","horizon",")","starting","from","any","BxT","state","(","sampled","from","the","buffer","for","the","train","batch",")",".","value_targets_t0_to_Hm1_BxT",":","The","computed","value","function","targets","of","the","shape","(","t0","to","H-1",",","BxT",")",".","Returns",":","The","total","actor","loss","tensor","."]
code: def _compute_actor_loss(
      self,
      *,
      module_id: ModuleID,
      config: DreamerV3Config,
      dream_data: Dict[str, TensorType],
      value_targets_t0_to_Hm1_BxT: TensorType,
  ) -> TensorType:
      """Helper method computing the actor's loss terms.
  
      Args:
          module_id: The module_id for which to compute the actor loss.
          config: The DreamerV3Config to use.
          dream_data: The data generated by dreaming for H steps (horizon) starting
              from any BxT state (sampled from the buffer for the train batch).
          value_targets_t0_to_Hm1_BxT: The computed value function targets of the
              shape (t0 to H-1, BxT).
  
      Returns:
          The total actor loss tensor.
      """
      actor = self.module[module_id].actor
  
      # Note: `scaled_value_targets_t0_to_Hm1_B` are NOT stop_gradient'd yet.
      scaled_value_targets_t0_to_Hm1_B = (
          self._compute_scaled_value_targets(
              module_id=module_id,
              config=config,
              value_targets_t0_to_Hm1_BxT=value_targets_t0_to_Hm1_BxT,
              value_predictions_t0_to_Hm1_BxT=dream_data[
                  "values_dreamed_t0_to_H_BxT"
              ][:-1],
          )
      )
  
      # Actions actually taken in the dream.
      actions_dreamed = tf.stop_gradient(
          dream_data["actions_dreamed_t0_to_H_BxT"]
      )[:-1]
      actions_dreamed_dist_params_t0_to_Hm1_B = dream_data[
          "actions_dreamed_dist_params_t0_to_H_BxT"
      ][:-1]
  
      dist_t0_to_Hm1_B = actor.get_action_dist_object(
          actions_dreamed_dist_params_t0_to_Hm1_B
      )
  
      # Compute log(p)s of all possible actions in the dream.
      if isinstance(
          self.module[module_id].actor.action_space, gym.spaces.Discrete
      ):
          # Note that when we create the Categorical action distributions, we compute
          # unimix probs, then math.log these and provide these log(p) as "logits" to
          # the Categorical. So here, we'll continue to work with log(p)s (not
          # really "logits")!
          logp_actions_t0_to_Hm1_B = (
              actions_dreamed_dist_params_t0_to_Hm1_B
          )
  
          # Log probs of actions actually taken in the dream.
          logp_actions_dreamed_t0_to_Hm1_B = tf.reduce_sum(
              actions_dreamed * logp_actions_t0_to_Hm1_B,
              axis=-1,
          )
          # First term of loss function. [1] eq. 11.
          logp_loss_H_B = (
              logp_actions_dreamed_t0_to_Hm1_B
              * tf.stop_gradient(scaled_value_targets_t0_to_Hm1_B)
          )
      # Box space.
      else:
          logp_actions_dreamed_t0_to_Hm1_B = dist_t0_to_Hm1_B.log_prob(
              actions_dreamed
          )
          # First term of loss function. [1] eq. 11.
          logp_loss_H_B = scaled_value_targets_t0_to_Hm1_B
  
      assert len(logp_loss_H_B.shape) == 2
  
      # Add entropy loss term (second term [1] eq. 11).
      entropy_H_B = dist_t0_to_Hm1_B.entropy()
      assert len(entropy_H_B.shape) == 2
      entropy = tf.reduce_mean(entropy_H_B)
  
      L_actor_reinforce_term_H_B = -logp_loss_H_B
      L_actor_action_entropy_term_H_B = (
          -config.entropy_scale * entropy_H_B
      )
  
      L_actor_H_B = (
          L_actor_reinforce_term_H_B + L_actor_action_entropy_term_H_B
      )
      # Mask out everything that goes beyond a predicted continue=False boundary.
      L_actor_H_B *= tf.stop_gradient(
          dream_data["dream_loss_weights_t0_to_H_BxT"]
      )[:-1]
      L_actor = tf.reduce_mean(L_actor_H_B)
  
      self.register_metrics(
          module_id,
          metrics_dict={
              "ACTOR_L_total": L_actor,
              "ACTOR_value_targets_pct95_ema": actor.ema_value_target_pct95,
              "ACTOR_value_targets_pct5_ema": actor.ema_value_target_pct5,
              "ACTOR_action_entropy": entropy,
              # Individual loss terms.
              "ACTOR_L_neglogp_reinforce_term": tf.reduce_mean(
                  L_actor_reinforce_term_H_B
              ),
              "ACTOR_L_neg_entropy_term": tf.reduce_mean(
                  L_actor_action_entropy_term_H_B
              ),
          },
      )
      if config.report_individual_batch_item_stats:
          self.register_metrics(
              module_id,
              metrics_dict={
                  "ACTOR_L_total_H_BxT": L_actor_H_B,
                  "ACTOR_logp_actions_dreamed_H_BxT": (
                      logp_actions_dreamed_t0_to_Hm1_B
                  ),
                  "ACTOR_scaled_value_targets_H_BxT": (
                      scaled_value_targets_t0_to_Hm1_B
                  ),
                  "ACTOR_action_entropy_H_BxT": entropy_H_B,
                  # Individual loss terms.
                  "ACTOR_L_neglogp_reinforce_term_H_BxT": L_actor_reinforce_term_H_B,
                  "ACTOR_L_neg_entropy_term_H_BxT": L_actor_action_entropy_term_H_B,
              },
          )
  
      return L_actor
  
imports: from typing import Any, Dict, Tuple
  import gymnasium
  from ray.rllib.algorithms.dreamerv3.dreamerv3 import DreamerV3Config
  from ray.rllib.algorithms.dreamerv3.dreamerv3_learner import DreamerV3Learner
  from ray.rllib.core.learner.learner import ParamDict
  from ray.rllib.core.learner.tf.tf_learner import TfLearner
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import deprecation_warning
  from ray.rllib.utils.framework import try_import_tf, try_import_tfp
  from ray.rllib.utils.tf_utils import symlog, two_hot, clip_gradients
  from ray.rllib.utils.typing import ModuleID, TensorType
label_desc: Use image node_id 7 for calling the DreamerV3TfLearner obj's underlying member method code with example usage: obj._compute_actor_loss() and returns: L_actor