class_docstr: Defines a configuration class from which a SlateQ Algorithm can be built.
  
  Example:
      >>> from rllib_slate_q.slate_q import SlateQConfig
      >>> config = SlateQConfig().training(lr=0.01).resources(num_gpus=1)
      >>> print(config.to_dict())  # doctest: +SKIP
      >>> # Build a Algorithm object from the config and run 1 training iteration.
      >>> algo = config.build(env="CartPole-v1")  # doctest: +SKIP
      >>> algo.train()  # doctest: +SKIP
  
  Example:
      >>> from rllib_slate_q.slate_q import SlateQConfig
      >>> from ray import air
      >>> from ray import tune
      >>> config = SlateQConfig()
      >>> # Print out some default values.
      >>> print(config.lr)  # doctest: +SKIP
      >>> # Update the config object.
      >>> config = config.training(  # doctest: +SKIP
      ...     lr=tune.grid_search([0.001, 0.0001]))
      >>> # Set the config object's env.
      >>> config = config.environment(env="CartPole-v1")  # doctest: +SKIP
      >>> # Use to_dict() to get the old-style python config dict
      >>> # when running with tune.
      >>> tune.Tuner(  # doctest: +SKIP
      ...     "SlateQ",
      ...     run_config=air.RunConfig(stop={"episode_reward_mean": 160.0}),
      ...     param_space=config.to_dict(),
      ... ).fit()
class_docstr_tok: ["Defines","a","configuration","class","from","which","a","SlateQ","Algorithm","can","be","built",".","Example",":",">",">",">","from","rllib_slate_q.slate_q","import","SlateQConfig",">",">",">","config","=","SlateQConfig","(",")",".training","(","lr=0.01",")",".resources","(","num_gpus=1",")",">",">",">","print","(","config.to_dict","(",")",")","#","doctest",":","+SKIP",">",">",">","#","Build","a","Algorithm","object","from","the","config","and","run","1","training","iteration",".",">",">",">","algo","=","config.build","(","env=","''","CartPole-v1","''",")","#","doctest",":","+SKIP",">",">",">","algo.train","(",")","#","doctest",":","+SKIP","Example",":",">",">",">","from","rllib_slate_q.slate_q","import","SlateQConfig",">",">",">","from","ray","import","air",">",">",">","from","ray","import","tune",">",">",">","config","=","SlateQConfig","(",")",">",">",">","#","Print","out","some","default","values",".",">",">",">","print","(","config.lr",")","#","doctest",":","+SKIP",">",">",">","#","Update","the","config","object",".",">",">",">","config","=","config.training","(","#","doctest",":","+SKIP","...","lr=tune.grid_search","(","[","0.001",",","0.0001","]",")",")",">",">",">","#","Set","the","config","object","'s","env",".",">",">",">","config","=","config.environment","(","env=","''","CartPole-v1","''",")","#","doctest",":","+SKIP",">",">",">","#","Use","to_dict","(",")","to","get","the","old-style","python","config","dict",">",">",">","#","when","running","with","tune",".",">",">",">","tune.Tuner","(","#","doctest",":","+SKIP","...","``","SlateQ","''",",","...","run_config=air.RunConfig","(","stop=","{","``","episode_reward_mean","''",":","160.0","}",")",",","...","param_space=config.to_dict","(",")",",","...",")",".fit","(",")"]
docstr: Sets the training related configuration.
  
  Args:
      replay_buffer_config: The config dict to specify the replay buffer used.
          May contain a `type` key (default: `MultiAgentPrioritizedReplayBuffer`)
          indicating the class being used. All other keys specify the names
          and values of kwargs passed to to this class' constructor.
      fcnet_hiddens_per_candidate: Dense-layer setup for each the n (document)
          candidate Q-network stacks.
      target_network_update_freq: Update the target network every
          `target_network_update_freq` sample steps.
      tau: Update the target by   au * policy + (1-       au) * target_policy.
      use_huber: If True, use huber loss instead of squared loss for critic
          network. Conventionally, no need to clip gradients if using a huber
          loss.
      huber_threshold: The threshold for the Huber loss.
      training_intensity: If set, this will fix the ratio of replayed from a
          buffer and learned on timesteps to sampled from an environment and
          stored in the replay buffer timesteps. Otherwise, the replay will
          proceed at the native ratio determined by
          `(train_batch_size / rollout_fragment_length)`.
      lr_schedule: Learning rate schedule. In the format of
          [[timestep, lr-value], [timestep, lr-value], ...]
          Intermediary timesteps will be assigned to interpolated learning rate
          values. A schedule should normally start from timestep 0.
      lr_choice_model: Learning rate for adam optimizer for the user choice model.
          So far, only relevant/supported for framework=torch.
      rmsprop_epsilon: RMSProp epsilon hyperparameter.
      grad_clip: If not None, clip gradients during optimization at this value.
      n_step: N-step parameter for Q-learning.
  
  Returns:
      This updated AlgorithmConfig object.
docstr_tok: ["Sets","the","training","related","configuration",".","Args",":","replay_buffer_config",":","The","config","dict","to","specify","the","replay","buffer","used",".","May","contain","a","`","type","`","key","(","default",":","`","MultiAgentPrioritizedReplayBuffer","`",")","indicating","the","class","being","used",".","All","other","keys","specify","the","names","and","values","of","kwargs","passed","to","to","this","class","'","constructor",".","fcnet_hiddens_per_candidate",":","Dense-layer","setup","for","each","the","n","(","document",")","candidate","Q-network","stacks",".","target_network_update_freq",":","Update","the","target","network","every","`","target_network_update_freq","`","sample","steps",".","tau",":","Update","the","target","by","au","*","policy","+","(","1-","au",")","*","target_policy",".","use_huber",":","If","True",",","use","huber","loss","instead","of","squared","loss","for","critic","network",".","Conventionally",",","no","need","to","clip","gradients","if","using","a","huber","loss",".","huber_threshold",":","The","threshold","for","the","Huber","loss",".","training_intensity",":","If","set",",","this","will","fix","the","ratio","of","replayed","from","a","buffer","and","learned","on","timesteps","to","sampled","from","an","environment","and","stored","in","the","replay","buffer","timesteps",".","Otherwise",",","the","replay","will","proceed","at","the","native","ratio","determined","by","`","(","train_batch_size","\/","rollout_fragment_length",")","`",".","lr_schedule",":","Learning","rate","schedule",".","In","the","format","of","[","[","timestep",",","lr-value","]",",","[","timestep",",","lr-value","]",",","...","]","Intermediary","timesteps","will","be","assigned","to","interpolated","learning","rate","values",".","A","schedule","should","normally","start","from","timestep","0.","lr_choice_model",":","Learning","rate","for","adam","optimizer","for","the","user","choice","model",".","So","far",",","only","relevant\/supported","for","framework=torch",".","rmsprop_epsilon",":","RMSProp","epsilon","hyperparameter",".","grad_clip",":","If","not","None",",","clip","gradients","during","optimization","at","this","value",".","n_step",":","N-step","parameter","for","Q-learning",".","Returns",":","This","updated","AlgorithmConfig","object","."]
code: def training(
      self,
      *,
      replay_buffer_config: Optional[Dict[str, Any]] = NotProvided,
      fcnet_hiddens_per_candidate: Optional[List[int]] = NotProvided,
      target_network_update_freq: Optional[int] = NotProvided,
      tau: Optional[float] = NotProvided,
      use_huber: Optional[bool] = NotProvided,
      huber_threshold: Optional[float] = NotProvided,
      training_intensity: Optional[float] = NotProvided,
      lr_schedule: Optional[
          List[List[Union[int, float]]]
      ] = NotProvided,
      lr_choice_model: Optional[bool] = NotProvided,
      rmsprop_epsilon: Optional[float] = NotProvided,
      grad_clip: Optional[float] = NotProvided,
      n_step: Optional[int] = NotProvided,
      num_steps_sampled_before_learning_starts: Optional[
          int
      ] = NotProvided,
      **kwargs,
  ) -> "SlateQConfig":
      """Sets the training related configuration.
  
      Args:
          replay_buffer_config: The config dict to specify the replay buffer used.
              May contain a `type` key (default: `MultiAgentPrioritizedReplayBuffer`)
              indicating the class being used. All other keys specify the names
              and values of kwargs passed to to this class' constructor.
          fcnet_hiddens_per_candidate: Dense-layer setup for each the n (document)
              candidate Q-network stacks.
          target_network_update_freq: Update the target network every
              `target_network_update_freq` sample steps.
          tau: Update the target by \tau * policy + (1-\tau) * target_policy.
          use_huber: If True, use huber loss instead of squared loss for critic
              network. Conventionally, no need to clip gradients if using a huber
              loss.
          huber_threshold: The threshold for the Huber loss.
          training_intensity: If set, this will fix the ratio of replayed from a
              buffer and learned on timesteps to sampled from an environment and
              stored in the replay buffer timesteps. Otherwise, the replay will
              proceed at the native ratio determined by
              `(train_batch_size / rollout_fragment_length)`.
          lr_schedule: Learning rate schedule. In the format of
              [[timestep, lr-value], [timestep, lr-value], ...]
              Intermediary timesteps will be assigned to interpolated learning rate
              values. A schedule should normally start from timestep 0.
          lr_choice_model: Learning rate for adam optimizer for the user choice model.
              So far, only relevant/supported for framework=torch.
          rmsprop_epsilon: RMSProp epsilon hyperparameter.
          grad_clip: If not None, clip gradients during optimization at this value.
          n_step: N-step parameter for Q-learning.
  
      Returns:
          This updated AlgorithmConfig object.
      """
      # Pass kwargs onto super's `training()` method.
      super().training(**kwargs)
  
      if replay_buffer_config is not NotProvided:
          self.replay_buffer_config.update(replay_buffer_config)
      if fcnet_hiddens_per_candidate is not NotProvided:
          self.fcnet_hiddens_per_candidate = fcnet_hiddens_per_candidate
      if target_network_update_freq is not NotProvided:
          self.target_network_update_freq = target_network_update_freq
      if tau is not NotProvided:
          self.tau = tau
      if use_huber is not NotProvided:
          self.use_huber = use_huber
      if huber_threshold is not NotProvided:
          self.huber_threshold = huber_threshold
      if training_intensity is not NotProvided:
          self.training_intensity = training_intensity
      if lr_schedule is not NotProvided:
          self.lr_schedule = lr_schedule
      if lr_choice_model is not NotProvided:
          self.lr_choice_model = lr_choice_model
      if rmsprop_epsilon is not NotProvided:
          self.rmsprop_epsilon = rmsprop_epsilon
      if grad_clip is not NotProvided:
          self.grad_clip = grad_clip
      if n_step is not NotProvided:
          self.n_step = n_step
      if num_steps_sampled_before_learning_starts is not NotProvided:
          self.num_steps_sampled_before_learning_starts = (
              num_steps_sampled_before_learning_starts
          )
  
      return self
  
file_path: ray/rllib_contrib/slate_q/src/rllib_slate_q/slate_q/slateq.py
imports: import logging
  from typing import Any, Dict, List, Optional, Type, Union
  from rllib_slate_q.slate_q.slateq_tf_policy import SlateQTFPolicy
  from rllib_slate_q.slate_q.slateq_torch_policy import SlateQTorchPolicy
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.algorithms.dqn.dqn import DQN
  from ray.rllib.policy.policy import Policy
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import DEPRECATED_VALUE
label_desc: Use image node_id 2 for calling the SlateQConfig obj's underlying member method code with example usage: obj.training() and returns: self