class_docstr: Defines a configuration class from which a DreamerV3 can be built.
  
  .. testcode::
  
      from ray.rllib.algorithms.dreamerv3 import DreamerV3Config
      config = (
          DreamerV3Config()
          .environment("CartPole-v1")
          .training(
              model_size="XS",
              training_ratio=1,
              # TODO
              model={
                  "batch_size_B": 1,
                  "batch_length_T": 1,
                  "horizon_H": 1,
                  "gamma": 0.997,
                  "model_size": "XS",
              },
          )
      )
  
      config = config.resources(num_learner_workers=0)
      # Build a Algorithm object from the config and run 1 training iteration.
      algo = config.build()
      # algo.train()
      del algo
  
  .. testoutput::
      :hide:
  
      ...
class_docstr_tok: ["Defines","a","configuration","class","from","which","a","DreamerV3","can","be","built",".","..","testcode",":",":","from","ray.rllib.algorithms.dreamerv3","import","DreamerV3Config","config","=","(","DreamerV3Config","(",")",".environment","(","``","CartPole-v1","''",")",".training","(","model_size=","''","XS","''",",","training_ratio=1",",","#","TODO","model=","{","``","batch_size_B","''",":","1",",","``","batch_length_T","''",":","1",",","``","horizon_H","''",":","1",",","``","gamma","''",":","0.997",",","``","model_size","''",":","``","XS","''",",","}",",",")",")","config","=","config.resources","(","num_learner_workers=0",")","#","Build","a","Algorithm","object","from","the","config","and","run","1","training","iteration",".","algo","=","config.build","(",")","#","algo.train","(",")","del","algo","..","testoutput",":",":",":","hide",":","..."]
docstr: Sets the training related configuration.
  
  Args:
      model_size: The main switch for adjusting the overall model size. See [1]
          (table B) for more information on the effects of this setting on the
          model architecture.
          Supported values are "XS", "S", "M", "L", "XL" (as per the paper), as
          well as, "nano", "micro", "mini", and "XXS" (for RLlib's
          implementation). See ray.rllib.algorithms.dreamerv3.utils.
          __init__.py for the details on what exactly each size does to the layer
          sizes, number of layers, etc..
      training_ratio: The ratio of total steps trained (sum of the sizes of all
          batches ever sampled from the replay buffer) over the total env steps
          taken (in the actual environment, not the dreamed one). For example,
          if the training_ratio is 1024 and the batch size is 1024, we would take
          1 env step for every training update: 1024 / 1. If the training ratio
          is 512 and the batch size is 1024, we would take 2 env steps and then
          perform a single training update (on a 1024 batch): 1024 / 2.
      gc_frequency_train_steps: The frequency (in training iterations) with which
          we perform a `gc.collect()` calls at the end of a `training_step`
          iteration. Doing this more often adds a (albeit very small) performance
          overhead, but prevents memory leaks from becoming harmful.
          TODO (sven): This might not be necessary anymore, but needs to be
           confirmed experimentally.
      batch_size_B: The batch size (B) interpreted as number of rows (each of
          length `batch_length_T`) to sample from the replay buffer in each
          iteration.
      batch_length_T: The batch length (T) interpreted as the length of each row
          sampled from the replay buffer in each iteration. Note that
          `batch_size_B` rows will be sampled in each iteration. Rows normally
          contain consecutive data (consecutive timesteps from the same episode),
          but there might be episode boundaries in a row as well.
      horizon_H: The horizon (in timesteps) used to create dreamed data from the
          world model, which in turn is used to train/update both actor- and
          critic networks.
      gae_lambda: The lambda parameter used for computing the GAE-style
          value targets for the actor- and critic losses.
      entropy_scale: The factor with which to multiply the entropy loss term
          inside the actor loss.
      return_normalization_decay: The decay value to use when computing the
          running EMA values for return normalization (used in the actor loss).
      train_critic: Whether to train the critic network. If False, `train_actor`
          must also be False (cannot train actor w/o training the critic).
      train_actor: Whether to train the actor network. If True, `train_critic`
          must also be True (cannot train actor w/o training the critic).
      intrinsic_rewards_scale: The factor to multiply intrinsic rewards with
          before adding them to the extrinsic (environment) rewards.
      world_model_lr: The learning rate or schedule for the world model optimizer.
      actor_lr: The learning rate or schedule for the actor optimizer.
      critic_lr: The learning rate or schedule for the critic optimizer.
      world_model_grad_clip_by_global_norm: World model grad clipping value
          (by global norm).
      critic_grad_clip_by_global_norm: Critic grad clipping value
          (by global norm).
      actor_grad_clip_by_global_norm: Actor grad clipping value (by global norm).
      symlog_obs: Whether to symlog observations or not. If set to "auto"
          (default), will check for the environment's observation space and then
          only symlog if not an image space.
      use_float16: Whether to train with mixed float16 precision. In this mode,
          model parameters are stored as float32, but all computations are
          performed in float16 space (except for losses and distribution params
          and outputs).
      replay_buffer_config: Replay buffer config.
          Only serves in DreamerV3 to set the capacity of the replay buffer.
          Note though that in the paper ([1]) a size of 1M is used for all
          benchmarks and there doesn't seem to be a good reason to change this
          parameter.
          Examples:
          {
          "type": "EpisodeReplayBuffer",
          "capacity": 100000,
          }
  
  Returns:
      This updated AlgorithmConfig object.
code: def training(
      self,
      *,
      model_size: Optional[str] = NotProvided,
      training_ratio: Optional[float] = NotProvided,
      gc_frequency_train_steps: Optional[int] = NotProvided,
      batch_size_B: Optional[int] = NotProvided,
      batch_length_T: Optional[int] = NotProvided,
      horizon_H: Optional[int] = NotProvided,
      gae_lambda: Optional[float] = NotProvided,
      entropy_scale: Optional[float] = NotProvided,
      return_normalization_decay: Optional[float] = NotProvided,
      train_critic: Optional[bool] = NotProvided,
      train_actor: Optional[bool] = NotProvided,
      intrinsic_rewards_scale: Optional[float] = NotProvided,
      world_model_lr: Optional[LearningRateOrSchedule] = NotProvided,
      actor_lr: Optional[LearningRateOrSchedule] = NotProvided,
      critic_lr: Optional[LearningRateOrSchedule] = NotProvided,
      world_model_grad_clip_by_global_norm: Optional[
          float
      ] = NotProvided,
      critic_grad_clip_by_global_norm: Optional[float] = NotProvided,
      actor_grad_clip_by_global_norm: Optional[float] = NotProvided,
      symlog_obs: Optional[Union[bool, str]] = NotProvided,
      use_float16: Optional[bool] = NotProvided,
      replay_buffer_config: Optional[dict] = NotProvided,
      use_curiosity: Optional[bool] = NotProvided,
      **kwargs,
  ) -> "DreamerV3Config":
      """Sets the training related configuration.
  
      Args:
          model_size: The main switch for adjusting the overall model size. See [1]
              (table B) for more information on the effects of this setting on the
              model architecture.
              Supported values are "XS", "S", "M", "L", "XL" (as per the paper), as
              well as, "nano", "micro", "mini", and "XXS" (for RLlib's
              implementation). See ray.rllib.algorithms.dreamerv3.utils.
              __init__.py for the details on what exactly each size does to the layer
              sizes, number of layers, etc..
          training_ratio: The ratio of total steps trained (sum of the sizes of all
              batches ever sampled from the replay buffer) over the total env steps
              taken (in the actual environment, not the dreamed one). For example,
              if the training_ratio is 1024 and the batch size is 1024, we would take
              1 env step for every training update: 1024 / 1. If the training ratio
              is 512 and the batch size is 1024, we would take 2 env steps and then
              perform a single training update (on a 1024 batch): 1024 / 2.
          gc_frequency_train_steps: The frequency (in training iterations) with which
              we perform a `gc.collect()` calls at the end of a `training_step`
              iteration. Doing this more often adds a (albeit very small) performance
              overhead, but prevents memory leaks from becoming harmful.
              TODO (sven): This might not be necessary anymore, but needs to be
               confirmed experimentally.
          batch_size_B: The batch size (B) interpreted as number of rows (each of
              length `batch_length_T`) to sample from the replay buffer in each
              iteration.
          batch_length_T: The batch length (T) interpreted as the length of each row
              sampled from the replay buffer in each iteration. Note that
              `batch_size_B` rows will be sampled in each iteration. Rows normally
              contain consecutive data (consecutive timesteps from the same episode),
              but there might be episode boundaries in a row as well.
          horizon_H: The horizon (in timesteps) used to create dreamed data from the
              world model, which in turn is used to train/update both actor- and
              critic networks.
          gae_lambda: The lambda parameter used for computing the GAE-style
              value targets for the actor- and critic losses.
          entropy_scale: The factor with which to multiply the entropy loss term
              inside the actor loss.
          return_normalization_decay: The decay value to use when computing the
              running EMA values for return normalization (used in the actor loss).
          train_critic: Whether to train the critic network. If False, `train_actor`
              must also be False (cannot train actor w/o training the critic).
          train_actor: Whether to train the actor network. If True, `train_critic`
              must also be True (cannot train actor w/o training the critic).
          intrinsic_rewards_scale: The factor to multiply intrinsic rewards with
              before adding them to the extrinsic (environment) rewards.
          world_model_lr: The learning rate or schedule for the world model optimizer.
          actor_lr: The learning rate or schedule for the actor optimizer.
          critic_lr: The learning rate or schedule for the critic optimizer.
          world_model_grad_clip_by_global_norm: World model grad clipping value
              (by global norm).
          critic_grad_clip_by_global_norm: Critic grad clipping value
              (by global norm).
          actor_grad_clip_by_global_norm: Actor grad clipping value (by global norm).
          symlog_obs: Whether to symlog observations or not. If set to "auto"
              (default), will check for the environment's observation space and then
              only symlog if not an image space.
          use_float16: Whether to train with mixed float16 precision. In this mode,
              model parameters are stored as float32, but all computations are
              performed in float16 space (except for losses and distribution params
              and outputs).
          replay_buffer_config: Replay buffer config.
              Only serves in DreamerV3 to set the capacity of the replay buffer.
              Note though that in the paper ([1]) a size of 1M is used for all
              benchmarks and there doesn't seem to be a good reason to change this
              parameter.
              Examples:
              {
              "type": "EpisodeReplayBuffer",
              "capacity": 100000,
              }
  
      Returns:
          This updated AlgorithmConfig object.
      """
      # Not fully supported/tested yet.
      if use_curiosity is not NotProvided:
          raise ValueError(
              "`DreamerV3Config.curiosity` is not fully supported and tested yet! "
              "It thus remains disabled for now."
          )
  
      # Pass kwargs onto super's `training()` method.
      super().training(**kwargs)
  
      if model_size is not NotProvided:
          self.model_size = model_size
      if training_ratio is not NotProvided:
          self.training_ratio = training_ratio
      if gc_frequency_train_steps is not NotProvided:
          self.gc_frequency_train_steps = gc_frequency_train_steps
      if batch_size_B is not NotProvided:
          self.batch_size_B = batch_size_B
      if batch_length_T is not NotProvided:
          self.batch_length_T = batch_length_T
      if horizon_H is not NotProvided:
          self.horizon_H = horizon_H
      if gae_lambda is not NotProvided:
          self.gae_lambda = gae_lambda
      if entropy_scale is not NotProvided:
          self.entropy_scale = entropy_scale
      if return_normalization_decay is not NotProvided:
          self.return_normalization_decay = return_normalization_decay
      if train_critic is not NotProvided:
          self.train_critic = train_critic
      if train_actor is not NotProvided:
          self.train_actor = train_actor
      if intrinsic_rewards_scale is not NotProvided:
          self.intrinsic_rewards_scale = intrinsic_rewards_scale
      if world_model_lr is not NotProvided:
          self.world_model_lr = world_model_lr
      if actor_lr is not NotProvided:
          self.actor_lr = actor_lr
      if critic_lr is not NotProvided:
          self.critic_lr = critic_lr
      if world_model_grad_clip_by_global_norm is not NotProvided:
          self.world_model_grad_clip_by_global_norm = (
              world_model_grad_clip_by_global_norm
          )
      if critic_grad_clip_by_global_norm is not NotProvided:
          self.critic_grad_clip_by_global_norm = (
              critic_grad_clip_by_global_norm
          )
      if actor_grad_clip_by_global_norm is not NotProvided:
          self.actor_grad_clip_by_global_norm = (
              actor_grad_clip_by_global_norm
          )
      if symlog_obs is not NotProvided:
          self.symlog_obs = symlog_obs
      if use_float16 is not NotProvided:
          self.use_float16 = use_float16
      if replay_buffer_config is not NotProvided:
          # Override entire `replay_buffer_config` if `type` key changes.
          # Update, if `type` key remains the same or is not specified.
          new_replay_buffer_config = deep_update(
              {"replay_buffer_config": self.replay_buffer_config},
              {"replay_buffer_config": replay_buffer_config},
              False,
              ["replay_buffer_config"],
              ["replay_buffer_config"],
          )
          self.replay_buffer_config = new_replay_buffer_config[
              "replay_buffer_config"
          ]
  
      return self
  
imports: import copy
  import gc
  import logging
  import tree
  from typing import Any, Dict, List, Optional, Union
  import gymnasium
  import numpy
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.algorithms.dreamerv3.dreamerv3_catalog import DreamerV3Catalog
  from ray.rllib.algorithms.dreamerv3.utils import do_symlog_obs
  from ray.rllib.algorithms.dreamerv3.utils.env_runner import DreamerV3EnvRunner
  from ray.rllib.algorithms.dreamerv3.utils.summaries import report_predicted_vs_sampled_obs, report_sampling_and_replay_buffer
  from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec
  from ray.rllib.models.catalog import MODEL_DEFAULTS
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils import deep_update
  from ray.rllib.utils.annotations import override, PublicAPI
  from ray.rllib.utils.framework import try_import_tf
  from ray.rllib.utils.numpy import one_hot
  from ray.rllib.utils.metrics import ALL_MODULES, GARBAGE_COLLECTION_TIMER, LEARN_ON_BATCH_TIMER, NUM_AGENT_STEPS_SAMPLED, NUM_AGENT_STEPS_TRAINED, NUM_ENV_STEPS_SAMPLED, NUM_ENV_STEPS_TRAINED, NUM_GRAD_UPDATES_LIFETIME, NUM_SYNCH_WORKER_WEIGHTS, NUM_TRAINING_STEP_CALLS_SINCE_LAST_SYNCH_WORKER_WEIGHTS, SAMPLE_TIMER, SYNCH_WORKER_WEIGHTS_TIMER
  from ray.rllib.utils.replay_buffers.episode_replay_buffer import EpisodeReplayBuffer
  from ray.rllib.utils.typing import LearningRateOrSchedule, ResultDict
label_desc: Use image node_id 4 for calling the DreamerV3Config obj's underlying member method code with example usage: obj.training() and returns: self