class_docstr: Implements DreamerV3 losses and gradient-based update logic in TensorFlow.
  
  The critic EMA-copy update step can be found in the `DreamerV3Learner` base class,
  as it is framework independent.
  
  We define 3 local TensorFlow optimizers for the sub components "world_model",
  "actor", and "critic". Each of these optimizers might use a different learning rate,
  epsilon parameter, and gradient clipping thresholds and procedures.
docstr: Helper method computing the value targets.
  
  All args are (H, BxT, ...) and in non-symlog'd (real) reward space.
  Non-symlog is important b/c log(a+b) != log(a) + log(b).
  See [1] eq. 8 and 10.
  Thus, targets are always returned in real (non-symlog'd space).
  They need to be re-symlog'd before computing the critic loss from them (b/c the
  critic produces predictions in symlog space).
  Note that the original B and T ranks together form the new batch dimension
  (folded into BxT) and the new time rank is the dream horizon (hence: [H, BxT]).
  
  Variable names nomenclature:
  `H`=1+horizon_H (start state + H steps dreamed),
  `BxT`=batch_size * batch_length (meaning the original trajectory time rank has
  been folded).
  
  Rewards, continues, and value predictions are all of shape [t0-H, BxT]
  (time-major), whereas returned targets are [t0 to H-1, B] (last timestep missing
  b/c the target value equals vf prediction in that location anyways.
  
  Args:
      config: The DreamerV3Config to use.
      rewards_t0_to_H_BxT: The reward predictor's predictions over the
          dreamed trajectory t0 to H (and for the batch BxT).
      intrinsic_rewards_t1_to_H_BxT: The predicted intrinsic rewards over the
          dreamed trajectory t0 to H (and for the batch BxT).
      continues_t0_to_H_BxT: The continue predictor's predictions over the
          dreamed trajectory t0 to H (and for the batch BxT).
      value_predictions_t0_to_H_BxT: The critic's value predictions over the
          dreamed trajectory t0 to H (and for the batch BxT).
  
  Returns:
      The value targets in the shape: [t0toH-1, BxT]. Note that the last step (H)
      does not require a value target as it matches the critic's value prediction
      anyways.
docstr_tok: ["Helper","method","computing","the","value","targets",".","All","args","are","(","H",",","BxT",",","...",")","and","in","non-symlog","'d","(","real",")","reward","space",".","Non-symlog","is","important","b\/c","log","(","a+b",")","!","=","log","(","a",")","+","log","(","b",")",".","See","[","1","]","eq",".","8","and","10",".","Thus",",","targets","are","always","returned","in","real","(","non-symlog","'d","space",")",".","They","need","to","be","re-symlog","'d","before","computing","the","critic","loss","from","them","(","b\/c","the","critic","produces","predictions","in","symlog","space",")",".","Note","that","the","original","B","and","T","ranks","together","form","the","new","batch","dimension","(","folded","into","BxT",")","and","the","new","time","rank","is","the","dream","horizon","(","hence",":","[","H",",","BxT","]",")",".","Variable","names","nomenclature",":","`","H","`","=1+horizon_H","(","start","state","+","H","steps","dreamed",")",",","`","BxT","`","=batch_size","*","batch_length","(","meaning","the","original","trajectory","time","rank","has","been","folded",")",".","Rewards",",","continues",",","and","value","predictions","are","all","of","shape","[","t0-H",",","BxT","]","(","time-major",")",",","whereas","returned","targets","are","[","t0","to","H-1",",","B","]","(","last","timestep","missing","b\/c","the","target","value","equals","vf","prediction","in","that","location","anyways",".","Args",":","config",":","The","DreamerV3Config","to","use",".","rewards_t0_to_H_BxT",":","The","reward","predictor","'s","predictions","over","the","dreamed","trajectory","t0","to","H","(","and","for","the","batch","BxT",")",".","intrinsic_rewards_t1_to_H_BxT",":","The","predicted","intrinsic","rewards","over","the","dreamed","trajectory","t0","to","H","(","and","for","the","batch","BxT",")",".","continues_t0_to_H_BxT",":","The","continue","predictor","'s","predictions","over","the","dreamed","trajectory","t0","to","H","(","and","for","the","batch","BxT",")",".","value_predictions_t0_to_H_BxT",":","The","critic","'s","value","predictions","over","the","dreamed","trajectory","t0","to","H","(","and","for","the","batch","BxT",")",".","Returns",":","The","value","targets","in","the","shape",":","[","t0toH-1",",","BxT","]",".","Note","that","the","last","step","(","H",")","does","not","require","a","value","target","as","it","matches","the","critic","'s","value","prediction","anyways","."]
code: def _compute_value_targets(
      self,
      *,
      config: DreamerV3Config,
      rewards_t0_to_H_BxT: TensorType,
      intrinsic_rewards_t1_to_H_BxT: TensorType,
      continues_t0_to_H_BxT: TensorType,
      value_predictions_t0_to_H_BxT: TensorType,
  ) -> TensorType:
      """Helper method computing the value targets.
  
      All args are (H, BxT, ...) and in non-symlog'd (real) reward space.
      Non-symlog is important b/c log(a+b) != log(a) + log(b).
      See [1] eq. 8 and 10.
      Thus, targets are always returned in real (non-symlog'd space).
      They need to be re-symlog'd before computing the critic loss from them (b/c the
      critic produces predictions in symlog space).
      Note that the original B and T ranks together form the new batch dimension
      (folded into BxT) and the new time rank is the dream horizon (hence: [H, BxT]).
  
      Variable names nomenclature:
      `H`=1+horizon_H (start state + H steps dreamed),
      `BxT`=batch_size * batch_length (meaning the original trajectory time rank has
      been folded).
  
      Rewards, continues, and value predictions are all of shape [t0-H, BxT]
      (time-major), whereas returned targets are [t0 to H-1, B] (last timestep missing
      b/c the target value equals vf prediction in that location anyways.
  
      Args:
          config: The DreamerV3Config to use.
          rewards_t0_to_H_BxT: The reward predictor's predictions over the
              dreamed trajectory t0 to H (and for the batch BxT).
          intrinsic_rewards_t1_to_H_BxT: The predicted intrinsic rewards over the
              dreamed trajectory t0 to H (and for the batch BxT).
          continues_t0_to_H_BxT: The continue predictor's predictions over the
              dreamed trajectory t0 to H (and for the batch BxT).
          value_predictions_t0_to_H_BxT: The critic's value predictions over the
              dreamed trajectory t0 to H (and for the batch BxT).
  
      Returns:
          The value targets in the shape: [t0toH-1, BxT]. Note that the last step (H)
          does not require a value target as it matches the critic's value prediction
          anyways.
      """
      # The first reward is irrelevant (not used for any VF target).
      rewards_t1_to_H_BxT = rewards_t0_to_H_BxT[1:]
      if intrinsic_rewards_t1_to_H_BxT is not None:
          rewards_t1_to_H_BxT += intrinsic_rewards_t1_to_H_BxT
  
      # In all the following, when building value targets for t=1 to T=H,
      # exclude rewards & continues for t=1 b/c we don't need r1 or c1.
      # The target (R1) for V1 is built from r2, c2, and V2/R2.
      discount = (
          continues_t0_to_H_BxT[1:] * config.gamma
      )  # shape=[2-16, BxT]
      Rs = [value_predictions_t0_to_H_BxT[-1]]  # Rs indices=[16]
      intermediates = (
          rewards_t1_to_H_BxT
          + discount
          * (1 - config.gae_lambda)
          * value_predictions_t0_to_H_BxT[1:]
      )
      # intermediates.shape=[2-16, BxT]
  
      # Loop through reversed timesteps (axis=1) from T+1 to t=2.
      for t in reversed(range(discount.shape[0])):
          Rs.append(
              intermediates[t]
              + discount[t] * config.gae_lambda * Rs[-1]
          )
  
      # Reverse along time axis and cut the last entry (value estimate at very end
      # cannot be learnt from as it's the same as the ... well ... value estimate).
      targets_t0toHm1_BxT = tf.stack(list(reversed(Rs))[:-1], axis=0)
      # targets.shape=[t0 to H-1,BxT]
  
      return targets_t0toHm1_BxT
  
imports: from typing import Any, Dict, Tuple
  import gymnasium
  from ray.rllib.algorithms.dreamerv3.dreamerv3 import DreamerV3Config
  from ray.rllib.algorithms.dreamerv3.dreamerv3_learner import DreamerV3Learner
  from ray.rllib.core.learner.learner import ParamDict
  from ray.rllib.core.learner.tf.tf_learner import TfLearner
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import deprecation_warning
  from ray.rllib.utils.framework import try_import_tf, try_import_tfp
  from ray.rllib.utils.tf_utils import symlog, two_hot, clip_gradients
  from ray.rllib.utils.typing import ModuleID, TensorType
label_desc: Use image node_id 9 for calling the DreamerV3TfLearner obj's underlying member method code with example usage: obj._compute_value_targets() and returns: targets_t0toHm1_BxT