class_docstr: Extension of standard TorchModelV2 to provide dueling-Q functionality.
class_docstr_tok: ["Extension","of","standard","TorchModelV2","to","provide","dueling-Q","functionality","."]
docstr: Initialize variables of this model.
  
  Extra model kwargs:
      q_hiddens (Sequence[int]): List of layer-sizes after(!) the
          Advantages(A)/Value(V)-split. Hence, each of the A- and V-
          branches will have this structure of Dense layers. To define
          the NN before this A/V-split, use - as always -
          config["model"]["fcnet_hiddens"].
      dueling: Whether to build the advantage(A)/value(V) heads
          for DDQN. If True, Q-values are calculated as:
          Q = (A - mean[A]) + V. If False, raw NN output is interpreted
          as Q-values.
      dueling_activation: The activation to use for all dueling
          layers (A- and V-branch). One of "relu", "tanh", "linear".
      num_atoms: If >1, enables distributional DQN.
      use_noisy: Use noisy layers.
      v_min: Min value support for distributional DQN.
      v_max: Max value support for distributional DQN.
      sigma0 (float): Initial value of noisy layers.
      add_layer_norm: Enable layer norm (for param noise).
docstr_tok: ["Initialize","variables","of","this","model",".","Extra","model","kwargs",":","q_hiddens","(","Sequence","[","int","]",")",":","List","of","layer-sizes","after","(","!",")","the","Advantages","(","A",")","\/Value","(","V",")","-split",".","Hence",",","each","of","the","A-","and","V-","branches","will","have","this","structure","of","Dense","layers",".","To","define","the","NN","before","this","A\/V-split",",","use","-","as","always","-","config","[","``","model","''","]","[","``","fcnet_hiddens","''","]",".","dueling",":","Whether","to","build","the","advantage","(","A",")","\/value","(","V",")","heads","for","DDQN",".","If","True",",","Q-values","are","calculated","as",":","Q","=","(","A","-","mean","[","A","]",")","+","V.","If","False",",","raw","NN","output","is","interpreted","as","Q-values",".","dueling_activation",":","The","activation","to","use","for","all","dueling","layers","(","A-","and","V-branch",")",".","One","of","``","relu","''",",","``","tanh","''",",","``","linear","''",".","num_atoms",":","If",">","1",",","enables","distributional","DQN",".","use_noisy",":","Use","noisy","layers",".","v_min",":","Min","value","support","for","distributional","DQN",".","v_max",":","Max","value","support","for","distributional","DQN",".","sigma0","(","float",")",":","Initial","value","of","noisy","layers",".","add_layer_norm",":","Enable","layer","norm","(","for","param","noise",")","."]
code: def __init__(
      self,
      obs_space: gym.spaces.Space,
      action_space: gym.spaces.Space,
      num_outputs: int,
      model_config: ModelConfigDict,
      name: str,
      *,
      q_hiddens: Sequence[int] = (256,),
      dueling: bool = False,
      dueling_activation: str = "relu",
      num_atoms: int = 1,
      use_noisy: bool = False,
      v_min: float = -10.0,
      v_max: float = 10.0,
      sigma0: float = 0.5,
      # TODO(sven): Move `add_layer_norm` into ModelCatalog as
      #  generic option, then error if we use ParameterNoise as
      #  Exploration type and do not have any LayerNorm layers in
      #  the net.
      add_layer_norm: bool = False,
  ):
      """Initialize variables of this model.
  
      Extra model kwargs:
          q_hiddens (Sequence[int]): List of layer-sizes after(!) the
              Advantages(A)/Value(V)-split. Hence, each of the A- and V-
              branches will have this structure of Dense layers. To define
              the NN before this A/V-split, use - as always -
              config["model"]["fcnet_hiddens"].
          dueling: Whether to build the advantage(A)/value(V) heads
              for DDQN. If True, Q-values are calculated as:
              Q = (A - mean[A]) + V. If False, raw NN output is interpreted
              as Q-values.
          dueling_activation: The activation to use for all dueling
              layers (A- and V-branch). One of "relu", "tanh", "linear".
          num_atoms: If >1, enables distributional DQN.
          use_noisy: Use noisy layers.
          v_min: Min value support for distributional DQN.
          v_max: Max value support for distributional DQN.
          sigma0 (float): Initial value of noisy layers.
          add_layer_norm: Enable layer norm (for param noise).
      """
      nn.Module.__init__(self)
      super(DQNTorchModel, self).__init__(
          obs_space, action_space, num_outputs, model_config, name
      )
  
      self.dueling = dueling
      self.num_atoms = num_atoms
      self.v_min = v_min
      self.v_max = v_max
      self.sigma0 = sigma0
      ins = num_outputs
  
      advantage_module = nn.Sequential()
      value_module = nn.Sequential()
  
      # Dueling case: Build the shared (advantages and value) fc-network.
      for i, n in enumerate(q_hiddens):
          if use_noisy:
              advantage_module.add_module(
                  "dueling_A_{}".format(i),
                  NoisyLayer(
                      ins,
                      n,
                      sigma0=self.sigma0,
                      activation=dueling_activation,
                  ),
              )
              value_module.add_module(
                  "dueling_V_{}".format(i),
                  NoisyLayer(
                      ins,
                      n,
                      sigma0=self.sigma0,
                      activation=dueling_activation,
                  ),
              )
          else:
              advantage_module.add_module(
                  "dueling_A_{}".format(i),
                  SlimFC(ins, n, activation_fn=dueling_activation),
              )
              value_module.add_module(
                  "dueling_V_{}".format(i),
                  SlimFC(ins, n, activation_fn=dueling_activation),
              )
              # Add LayerNorm after each Dense.
              if add_layer_norm:
                  advantage_module.add_module(
                      "LayerNorm_A_{}".format(i), nn.LayerNorm(n)
                  )
                  value_module.add_module(
                      "LayerNorm_V_{}".format(i), nn.LayerNorm(n)
                  )
          ins = n
  
      # Actual Advantages layer (nodes=num-actions).
      if use_noisy:
          advantage_module.add_module(
              "A",
              NoisyLayer(
                  ins,
                  self.action_space.n * self.num_atoms,
                  sigma0,
                  activation=None,
              ),
          )
      elif q_hiddens:
          advantage_module.add_module(
              "A",
              SlimFC(
                  ins,
                  action_space.n * self.num_atoms,
                  activation_fn=None,
              ),
          )
  
      self.advantage_module = advantage_module
  
      # Value layer (nodes=1).
      if self.dueling:
          if use_noisy:
              value_module.add_module(
                  "V",
                  NoisyLayer(
                      ins, self.num_atoms, sigma0, activation=None
                  ),
              )
          elif q_hiddens:
              value_module.add_module(
                  "V", SlimFC(ins, self.num_atoms, activation_fn=None)
              )
          self.value_module = value_module
  
file_path: ray/python/ray/rllib/algorithms/dqn/dqn_torch_model.py
imports: from typing import Sequence
  import gymnasium
  from ray.rllib.models.torch.misc import SlimFC
  from ray.rllib.models.torch.modules.noisy_layer import NoisyLayer
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.typing import ModelConfigDict
label_desc: Use image node_id 1 to create a new DQNTorchModel object from inherited base classes: TorchModelV2, nn with example: obj = DQNTorchModel(obs_space, action_space, num_outputs, model_config, name)