class_docstr: Defines a configuration class from which an ApexDQN Algorithm can be built.
  
  Example:
      >>> from ray.rllib.algorithms.apex_dqn.apex_dqn import ApexDQNConfig
      >>> config = ApexDQNConfig()
      >>> print(config.replay_buffer_config) # doctest: +SKIP
      >>> replay_config = config.replay_buffer_config.update( # doctest: +SKIP
      ...     {
      ...         "capacity": 100000,
      ...         "prioritized_replay_alpha": 0.45,
      ...         "prioritized_replay_beta": 0.55,
      ...         "prioritized_replay_eps": 3e-6,
      ...     }
      ... )
      >>> config = config.training(replay_buffer_config=replay_config) #doctest: +SKIP
      >>> config = config.resources(num_gpus=1)  # doctest: +SKIP
      >>> config = config.rollouts(num_rollout_workers=30)  # doctest: +SKIP
      >>> config = config.environment("CartPole-v1")  # doctest: +SKIP
      >>> algo = config.build() # doctest: +SKIP
      >>> algo.train()  # doctest: +SKIP
  
  Example:
      >>> from ray.rllib.algorithms.apex_dqn.apex_dqn import ApexDQNConfig
      >>> from ray import air
      >>> from ray import tune
      >>> config = ApexDQNConfig()
      >>> config.training(  # doctest: +SKIP
      ...     num_atoms=tune.grid_search(list(range(1, 11)))
      >>> config.environment(env="CartPole-v1")  # doctest: +SKIP
      >>> tune.Tuner( # doctest: +SKIP
      ...     "APEX",
      ...     run_config=air.RunConfig(stop={"episode_reward_mean":200}),
      ...     param_space=config.to_dict()
      ... ).fit()
  
  Example:
      >>> from ray.rllib.algorithms.apex_dqn.apex_dqn import ApexDQNConfig
      >>> config = ApexDQNConfig()
      >>> print(config.exploration_config)  # doctest: +SKIP
      >>> explore_config = config.exploration_config.update(  # doctest: +SKIP
      ...     {
      ...         "type": "EpsilonGreedy",
      ...         "initial_epsilon": 0.96,
      ...         "final_epsilon": 0.01,
      ...         "epsilone_timesteps": 5000,
      ...     }
      ... )
      >>> config = config.training(  # doctest: +SKIP
      ...     lr_schedule=[[1, 1e-3, [500, 5e-3]]
      ... )
      >>> config = config.exploration(  # doctest: +SKIP
      ...     exploration_config=explore_config
      ... )
  
  Example:
      >>> from ray.rllib.algorithms.apex_dqn.apex_dqn import ApexDQNConfig
      >>> config = ApexDQNConfig()
      >>> print(config.exploration_config)  # doctest: +SKIP
      >>> explore_config = config.exploration_config.update(  # doctest: +SKIP
      ...     {
      ...         "type": "SoftQ",
      ...         "temperature": [1.0],
      ...     }
      ... )
      >>> config = config.training(  # doctest: +SKIP
      ...     lr_schedule=[[1, 1e-3, [500, 5e-3]]
      ... )
      >>> config = config.exploration(  # doctest: +SKIP
      ...     exploration_config=explore_config
      ... )
class_docstr_tok: ["Defines","a","configuration","class","from","which","an","ApexDQN","Algorithm","can","be","built",".","Example",":",">",">",">","from","ray.rllib.algorithms.apex_dqn.apex_dqn","import","ApexDQNConfig",">",">",">","config","=","ApexDQNConfig","(",")",">",">",">","print","(","config.replay_buffer_config",")","#","doctest",":","+SKIP",">",">",">","replay_config","=","config.replay_buffer_config.update","(","#","doctest",":","+SKIP","...","{","...","``","capacity","''",":","100000",",","...","``","prioritized_replay_alpha","''",":","0.45",",","...","``","prioritized_replay_beta","''",":","0.55",",","...","``","prioritized_replay_eps","''",":","3e-6",",","...","}","...",")",">",">",">","config","=","config.training","(","replay_buffer_config=replay_config",")","#","doctest",":","+SKIP",">",">",">","config","=","config.resources","(","num_gpus=1",")","#","doctest",":","+SKIP",">",">",">","config","=","config.rollouts","(","num_rollout_workers=30",")","#","doctest",":","+SKIP",">",">",">","config","=","config.environment","(","``","CartPole-v1","''",")","#","doctest",":","+SKIP",">",">",">","algo","=","config.build","(",")","#","doctest",":","+SKIP",">",">",">","algo.train","(",")","#","doctest",":","+SKIP","Example",":",">",">",">","from","ray.rllib.algorithms.apex_dqn.apex_dqn","import","ApexDQNConfig",">",">",">","from","ray","import","air",">",">",">","from","ray","import","tune",">",">",">","config","=","ApexDQNConfig","(",")",">",">",">","config.training","(","#","doctest",":","+SKIP","...","num_atoms=tune.grid_search","(","list","(","range","(","1",",","11",")",")",")",">",">",">","config.environment","(","env=","''","CartPole-v1","''",")","#","doctest",":","+SKIP",">",">",">","tune.Tuner","(","#","doctest",":","+SKIP","...","``","APEX","''",",","...","run_config=air.RunConfig","(","stop=","{","``","episode_reward_mean","''",":200","}",")",",","...","param_space=config.to_dict","(",")","...",")",".fit","(",")","Example",":",">",">",">","from","ray.rllib.algorithms.apex_dqn.apex_dqn","import","ApexDQNConfig",">",">",">","config","=","ApexDQNConfig","(",")",">",">",">","print","(","config.exploration_config",")","#","doctest",":","+SKIP",">",">",">","explore_config","=","config.exploration_config.update","(","#","doctest",":","+SKIP","...","{","...","``","type","''",":","``","EpsilonGreedy","''",",","...","``","initial_epsilon","''",":","0.96",",","...","``","final_epsilon","''",":","0.01",",","...","``","epsilone_timesteps","''",":","5000",",","...","}","...",")",">",">",">","config","=","config.training","(","#","doctest",":","+SKIP","...","lr_schedule=","[","[","1",",","1e-3",",","[","500",",","5e-3","]","]","...",")",">",">",">","config","=","config.exploration","(","#","doctest",":","+SKIP","...","exploration_config=explore_config","...",")","Example",":",">",">",">","from","ray.rllib.algorithms.apex_dqn.apex_dqn","import","ApexDQNConfig",">",">",">","config","=","ApexDQNConfig","(",")",">",">",">","print","(","config.exploration_config",")","#","doctest",":","+SKIP",">",">",">","explore_config","=","config.exploration_config.update","(","#","doctest",":","+SKIP","...","{","...","``","type","''",":","``","SoftQ","''",",","...","``","temperature","''",":","[","1.0","]",",","...","}","...",")",">",">",">","config","=","config.training","(","#","doctest",":","+SKIP","...","lr_schedule=","[","[","1",",","1e-3",",","[","500",",","5e-3","]","]","...",")",">",">",">","config","=","config.exploration","(","#","doctest",":","+SKIP","...","exploration_config=explore_config","...",")"]
docstr: Sets the training related configuration.
  
  Args:
      num_atoms: Number of atoms for representing the distribution of return.
          When this is greater than 1, distributional Q-learning is used.
      v_min: Minimum value estimation
      v_max: Maximum value estimation
      noisy: Whether to use noisy network to aid exploration. This adds parametric
          noise to the model weights.
      sigma0: Control the initial parameter noise for noisy nets.
      dueling: Whether to use dueling DQN.
      hiddens: Dense-layer setup for each the advantage branch and the value
          branch
      double_q: Whether to use double DQN.
      n_step: N-step for Q-learning.
      before_learn_on_batch: Callback to run before learning on a multi-agent
          batch of experiences.
      training_intensity: The intensity with which to update the model (vs
          collecting samples from the env).
          If None, uses "natural" values of:
          `train_batch_size` / (`rollout_fragment_length` x `num_workers` x
          `num_envs_per_worker`).
          If not None, will make sure that the ratio between timesteps inserted
          into and sampled from the buffer matches the given values.
          Example:
          training_intensity=1000.0
          train_batch_size=250
          rollout_fragment_length=1
          num_workers=1 (or 0)
          num_envs_per_worker=1
          -> natural value = 250 / 1 = 250.0
          -> will make sure that replay+train op will be executed 4x asoften as
          rollout+insert op (4 * 250 = 1000).
          See: rllib/algorithms/dqn/dqn.py::calculate_rr_weights for further
          details.
      replay_buffer_config: Replay buffer config.
          Examples:
          {
          "_enable_replay_buffer_api": True,
          "type": "MultiAgentReplayBuffer",
          "capacity": 50000,
          "replay_sequence_length": 1,
          }
          - OR -
          {
          "_enable_replay_buffer_api": True,
          "type": "MultiAgentPrioritizedReplayBuffer",
          "capacity": 50000,
          "prioritized_replay_alpha": 0.6,
          "prioritized_replay_beta": 0.4,
          "prioritized_replay_eps": 1e-6,
          "replay_sequence_length": 1,
          }
          - Where -
          prioritized_replay_alpha: Alpha parameter controls the degree of
          prioritization in the buffer. In other words, when a buffer sample has
          a higher temporal-difference error, with how much more probability
          should it drawn to use to update the parametrized Q-network. 0.0
          corresponds to uniform probability. Setting much above 1.0 may quickly
          result as the sampling distribution could become heavily “pointy” with
          low entropy.
          prioritized_replay_beta: Beta parameter controls the degree of
          importance sampling which suppresses the influence of gradient updates
          from samples that have higher probability of being sampled via alpha
          parameter and the temporal-difference error.
          prioritized_replay_eps: Epsilon parameter sets the baseline probability
          for sampling so that when the temporal-difference error of a sample is
          zero, there is still a chance of drawing the sample.
      max_requests_in_flight_per_replay_worker: Max number of inflight requests
          to each replay (shard) worker. See the FaultTolerantActorManager class
          for more details.
          Tuning these values is important when running experimens
          with large sample batches, where there is the risk that the object store
          may fill up, causing spilling of objects to disk. This can cause any
          asynchronous requests to become very slow, making your experiment run
          slow as well. You can inspect the object store during your experiment
          via a call to ray memory on your headnode, and by using the ray
          dashboard. If you're seeing that the object store is filling up,
          turn down the number of remote requests in flight, or enable compression
          in your experiment of timesteps.
      timeout_s_sampler_manager: The timeout for waiting for sampling results
          for workers -- typically if this is too low, the manager won't be able
          to retrieve ready sampling results.
      timeout_s_replay_manager: The timeout for waiting for replay worker
          results -- typically if this is too low, the manager won't be able to
          retrieve ready replay requests.
docstr_tok: ["Sets","the","training","related","configuration",".","Args",":","num_atoms",":","Number","of","atoms","for","representing","the","distribution","of","return",".","When","this","is","greater","than","1",",","distributional","Q-learning","is","used",".","v_min",":","Minimum","value","estimation","v_max",":","Maximum","value","estimation","noisy",":","Whether","to","use","noisy","network","to","aid","exploration",".","This","adds","parametric","noise","to","the","model","weights",".","sigma0",":","Control","the","initial","parameter","noise","for","noisy","nets",".","dueling",":","Whether","to","use","dueling","DQN",".","hiddens",":","Dense-layer","setup","for","each","the","advantage","branch","and","the","value","branch","double_q",":","Whether","to","use","double","DQN",".","n_step",":","N-step","for","Q-learning",".","before_learn_on_batch",":","Callback","to","run","before","learning","on","a","multi-agent","batch","of","experiences",".","training_intensity",":","The","intensity","with","which","to","update","the","model","(","vs","collecting","samples","from","the","env",")",".","If","None",",","uses","``","natural","''","values","of",":","`","train_batch_size","`","\/","(","`","rollout_fragment_length","`","x","`","num_workers","`","x","`","num_envs_per_worker","`",")",".","If","not","None",",","will","make","sure","that","the","ratio","between","timesteps","inserted","into","and","sampled","from","the","buffer","matches","the","given","values",".","Example",":","training_intensity=1000.0","train_batch_size=250","rollout_fragment_length=1","num_workers=1","(","or","0",")","num_envs_per_worker=1","-",">","natural","value","=","250","\/","1","=","250.0","-",">","will","make","sure","that","replay+train","op","will","be","executed","4x","asoften","as","rollout+insert","op","(","4","*","250","=","1000",")",".","See",":","rllib\/algorithms\/dqn\/dqn.py",":",":calculate_rr_weights","for","further","details",".","replay_buffer_config",":","Replay","buffer","config",".","Examples",":","{","``","_enable_replay_buffer_api","''",":","True",",","``","type","''",":","``","MultiAgentReplayBuffer","''",",","``","capacity","''",":","50000",",","``","replay_sequence_length","''",":","1",",","}","-","OR","-","{","``","_enable_replay_buffer_api","''",":","True",",","``","type","''",":","``","MultiAgentPrioritizedReplayBuffer","''",",","``","capacity","''",":","50000",",","``","prioritized_replay_alpha","''",":","0.6",",","``","prioritized_replay_beta","''",":","0.4",",","``","prioritized_replay_eps","''",":","1e-6",",","``","replay_sequence_length","''",":","1",",","}","-","Where","-","prioritized_replay_alpha",":","Alpha","parameter","controls","the","degree","of","prioritization","in","the","buffer",".","In","other","words",",","when","a","buffer","sample","has","a","higher","temporal-difference","error",",","with","how","much","more","probability","should","it","drawn","to","use","to","update","the","parametrized","Q-network",".","0.0","corresponds","to","uniform","probability",".","Setting","much","above","1.0","may","quickly","result","as","the","sampling","distribution","could","become","heavily","\u201c","pointy","\u201d","with","low","entropy",".","prioritized_replay_beta",":","Beta","parameter","controls","the","degree","of","importance","sampling","which","suppresses","the","influence","of","gradient","updates","from","samples","that","have","higher","probability","of","being","sampled","via","alpha","parameter","and","the","temporal-difference","error",".","prioritized_replay_eps",":","Epsilon","parameter","sets","the","baseline","probability","for","sampling","so","that","when","the","temporal-difference","error","of","a","sample","is","zero",",","there","is","still","a","chance","of","drawing","the","sample",".","max_requests_in_flight_per_replay_worker",":","Max","number","of","inflight","requests","to","each","replay","(","shard",")","worker",".","See","the","FaultTolerantActorManager","class","for","more","details",".","Tuning","these","values","is","important","when","running","experimens","with","large","sample","batches",",","where","there","is","the","risk","that","the","object","store","may","fill","up",",","causing","spilling","of","objects","to","disk",".","This","can","cause","any","asynchronous","requests","to","become","very","slow",",","making","your","experiment","run","slow","as","well",".","You","can","inspect","the","object","store","during","your","experiment","via","a","call","to","ray","memory","on","your","headnode",",","and","by","using","the","ray","dashboard",".","If","you","'re","seeing","that","the","object","store","is","filling","up",",","turn","down","the","number","of","remote","requests","in","flight",",","or","enable","compression","in","your","experiment","of","timesteps",".","timeout_s_sampler_manager",":","The","timeout","for","waiting","for","sampling","results","for","workers","--","typically","if","this","is","too","low",",","the","manager","wo","n't","be","able","to","retrieve","ready","sampling","results",".","timeout_s_replay_manager",":","The","timeout","for","waiting","for","replay","worker","results","--","typically","if","this","is","too","low",",","the","manager","wo","n't","be","able","to","retrieve","ready","replay","requests","."]
code: def training(
      self,
      *,
      max_requests_in_flight_per_replay_worker: Optional[
          int
      ] = NotProvided,
      timeout_s_sampler_manager: Optional[float] = NotProvided,
      timeout_s_replay_manager: Optional[float] = NotProvided,
      **kwargs,
  ) -> "ApexDQNConfig":
      """Sets the training related configuration.
  
      Args:
          num_atoms: Number of atoms for representing the distribution of return.
              When this is greater than 1, distributional Q-learning is used.
          v_min: Minimum value estimation
          v_max: Maximum value estimation
          noisy: Whether to use noisy network to aid exploration. This adds parametric
              noise to the model weights.
          sigma0: Control the initial parameter noise for noisy nets.
          dueling: Whether to use dueling DQN.
          hiddens: Dense-layer setup for each the advantage branch and the value
              branch
          double_q: Whether to use double DQN.
          n_step: N-step for Q-learning.
          before_learn_on_batch: Callback to run before learning on a multi-agent
              batch of experiences.
          training_intensity: The intensity with which to update the model (vs
              collecting samples from the env).
              If None, uses "natural" values of:
              `train_batch_size` / (`rollout_fragment_length` x `num_workers` x
              `num_envs_per_worker`).
              If not None, will make sure that the ratio between timesteps inserted
              into and sampled from the buffer matches the given values.
              Example:
              training_intensity=1000.0
              train_batch_size=250
              rollout_fragment_length=1
              num_workers=1 (or 0)
              num_envs_per_worker=1
              -> natural value = 250 / 1 = 250.0
              -> will make sure that replay+train op will be executed 4x asoften as
              rollout+insert op (4 * 250 = 1000).
              See: rllib/algorithms/dqn/dqn.py::calculate_rr_weights for further
              details.
          replay_buffer_config: Replay buffer config.
              Examples:
              {
              "_enable_replay_buffer_api": True,
              "type": "MultiAgentReplayBuffer",
              "capacity": 50000,
              "replay_sequence_length": 1,
              }
              - OR -
              {
              "_enable_replay_buffer_api": True,
              "type": "MultiAgentPrioritizedReplayBuffer",
              "capacity": 50000,
              "prioritized_replay_alpha": 0.6,
              "prioritized_replay_beta": 0.4,
              "prioritized_replay_eps": 1e-6,
              "replay_sequence_length": 1,
              }
              - Where -
              prioritized_replay_alpha: Alpha parameter controls the degree of
              prioritization in the buffer. In other words, when a buffer sample has
              a higher temporal-difference error, with how much more probability
              should it drawn to use to update the parametrized Q-network. 0.0
              corresponds to uniform probability. Setting much above 1.0 may quickly
              result as the sampling distribution could become heavily “pointy” with
              low entropy.
              prioritized_replay_beta: Beta parameter controls the degree of
              importance sampling which suppresses the influence of gradient updates
              from samples that have higher probability of being sampled via alpha
              parameter and the temporal-difference error.
              prioritized_replay_eps: Epsilon parameter sets the baseline probability
              for sampling so that when the temporal-difference error of a sample is
              zero, there is still a chance of drawing the sample.
          max_requests_in_flight_per_replay_worker: Max number of inflight requests
              to each replay (shard) worker. See the FaultTolerantActorManager class
              for more details.
              Tuning these values is important when running experimens
              with large sample batches, where there is the risk that the object store
              may fill up, causing spilling of objects to disk. This can cause any
              asynchronous requests to become very slow, making your experiment run
              slow as well. You can inspect the object store during your experiment
              via a call to ray memory on your headnode, and by using the ray
              dashboard. If you're seeing that the object store is filling up,
              turn down the number of remote requests in flight, or enable compression
              in your experiment of timesteps.
          timeout_s_sampler_manager: The timeout for waiting for sampling results
              for workers -- typically if this is too low, the manager won't be able
              to retrieve ready sampling results.
          timeout_s_replay_manager: The timeout for waiting for replay worker
              results -- typically if this is too low, the manager won't be able to
              retrieve ready replay requests.
      """
      # Pass kwargs onto super's `training()` method.
      super().training(**kwargs)
  
      if max_requests_in_flight_per_replay_worker is not NotProvided:
          self.max_requests_in_flight_per_replay_worker = (
              max_requests_in_flight_per_replay_worker
          )
      if timeout_s_sampler_manager is not NotProvided:
          self.timeout_s_sampler_manager = timeout_s_sampler_manager
      if timeout_s_replay_manager is not NotProvided:
          self.timeout_s_replay_manager = timeout_s_replay_manager
  
      return self
  
imports: import copy
  import platform
  import random
  from collections import defaultdict
  from typing import Any, Dict, List, Optional, Tuple, Union
  import ray
  from ray._private.dict import merge_dicts
  from ray.actor import ActorHandle
  from ray.rllib.algorithms import Algorithm
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.algorithms.dqn.dqn import DQN, DQNConfig
  from ray.rllib.algorithms.dqn.learner_thread import LearnerThread
  from ray.rllib.evaluation.rollout_worker import RolloutWorker
  from ray.rllib.evaluation.worker_set import handle_remote_call_result_errors
  from ray.rllib.utils.actor_manager import FaultTolerantActorManager
  from ray.rllib.utils.actors import create_colocated_actors
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import DEPRECATED_VALUE
  from ray.rllib.utils.metrics import LAST_TARGET_UPDATE_TS, NUM_AGENT_STEPS_SAMPLED, NUM_AGENT_STEPS_TRAINED, NUM_ENV_STEPS_SAMPLED, NUM_ENV_STEPS_TRAINED, NUM_TARGET_UPDATES, SAMPLE_TIMER, SYNCH_WORKER_WEIGHTS_TIMER, TARGET_NET_UPDATE_TIMER
  from ray.rllib.utils.typing import PartialAlgorithmConfigDict, ResultDict, SampleBatchType
  from ray.tune.execution.placement_groups import PlacementGroupFactory
  from ray.tune.trainable import Trainable
label_desc: Use image node_id 2 for calling the ApexDQNConfig obj's underlying member method code with example usage: obj.training() and returns: self