class_docstr: Implementation of:
  [1] Curiosity-driven Exploration by Self-supervised Prediction
  Pathak, Agrawal, Efros, and Darrell - UC Berkeley - ICML 2017.
  https://arxiv.org/pdf/1705.05363.pdf
  
  Learns a simplified model of the environment based on three networks:
  1) Embedding observations into latent space ("feature" network).
  2) Predicting the action, given two consecutive embedded observations
  ("inverse" network).
  3) Predicting the next embedded obs, given an obs and action
  ("forward" network).
  
  The less the agent is able to predict the actually observed next feature
  vector, given obs and action (through the forwards network), the larger the
  "intrinsic reward", which will be added to the extrinsic reward.
  Therefore, if a state transition was unexpected, the agent becomes
  "curious" and will further explore this transition leading to better
  exploration in sparse rewards environments.
docstr: Initializes a Curiosity object.
  
  Uses as defaults the hyperparameters described in [1].
  
  Args:
       feature_dim: The dimensionality of the feature (phi)
          vectors.
       feature_net_config: Optional model
          configuration for the feature network, producing feature
          vectors (phi) from observations. This can be used to configure
          fcnet- or conv_net setups to properly process any observation
          space.
       inverse_net_hiddens: Tuple of the layer sizes of the
          inverse (action predicting) NN head (on top of the feature
          outputs for phi and phi').
       inverse_net_activation: Activation specifier for the inverse
          net.
       forward_net_hiddens: Tuple of the layer sizes of the
          forward (phi' predicting) NN head.
       forward_net_activation: Activation specifier for the forward
          net.
       beta: Weight for the forward loss (over the inverse loss,
          which gets weight=1.0-beta) in the common loss term.
       eta: Weight for intrinsic rewards before being added to
          extrinsic ones.
       lr: The learning rate for the curiosity-specific
          optimizer, optimizing feature-, inverse-, and forward nets.
       sub_exploration: The config dict for
          the underlying Exploration to use (e.g. epsilon-greedy for
          DQN). If None, uses the FromSpecDict provided in the Policy's
          default config.
docstr_tok: ["Initializes","a","Curiosity","object",".","Uses","as","defaults","the","hyperparameters","described","in","[","1","]",".","Args",":","feature_dim",":","The","dimensionality","of","the","feature","(","phi",")","vectors",".","feature_net_config",":","Optional","model","configuration","for","the","feature","network",",","producing","feature","vectors","(","phi",")","from","observations",".","This","can","be","used","to","configure","fcnet-","or","conv_net","setups","to","properly","process","any","observation","space",".","inverse_net_hiddens",":","Tuple","of","the","layer","sizes","of","the","inverse","(","action","predicting",")","NN","head","(","on","top","of","the","feature","outputs","for","phi","and","phi","'",")",".","inverse_net_activation",":","Activation","specifier","for","the","inverse","net",".","forward_net_hiddens",":","Tuple","of","the","layer","sizes","of","the","forward","(","phi","'","predicting",")","NN","head",".","forward_net_activation",":","Activation","specifier","for","the","forward","net",".","beta",":","Weight","for","the","forward","loss","(","over","the","inverse","loss",",","which","gets","weight=1.0-beta",")","in","the","common","loss","term",".","eta",":","Weight","for","intrinsic","rewards","before","being","added","to","extrinsic","ones",".","lr",":","The","learning","rate","for","the","curiosity-specific","optimizer",",","optimizing","feature-",",","inverse-",",","and","forward","nets",".","sub_exploration",":","The","config","dict","for","the","underlying","Exploration","to","use","(","e.g",".","epsilon-greedy","for","DQN",")",".","If","None",",","uses","the","FromSpecDict","provided","in","the","Policy's","default","config","."]
code: def __init__(
      self,
      action_space: Space,
      *,
      framework: str,
      model: ModelV2,
      feature_dim: int = 288,
      feature_net_config: Optional[ModelConfigDict] = None,
      inverse_net_hiddens: Tuple[int] = (256,),
      inverse_net_activation: str = "relu",
      forward_net_hiddens: Tuple[int] = (256,),
      forward_net_activation: str = "relu",
      beta: float = 0.2,
      eta: float = 1.0,
      lr: float = 1e-3,
      sub_exploration: Optional[FromConfigSpec] = None,
      **kwargs,
  ):
      """Initializes a Curiosity object.
  
      Uses as defaults the hyperparameters described in [1].
  
      Args:
           feature_dim: The dimensionality of the feature (phi)
              vectors.
           feature_net_config: Optional model
              configuration for the feature network, producing feature
              vectors (phi) from observations. This can be used to configure
              fcnet- or conv_net setups to properly process any observation
              space.
           inverse_net_hiddens: Tuple of the layer sizes of the
              inverse (action predicting) NN head (on top of the feature
              outputs for phi and phi').
           inverse_net_activation: Activation specifier for the inverse
              net.
           forward_net_hiddens: Tuple of the layer sizes of the
              forward (phi' predicting) NN head.
           forward_net_activation: Activation specifier for the forward
              net.
           beta: Weight for the forward loss (over the inverse loss,
              which gets weight=1.0-beta) in the common loss term.
           eta: Weight for intrinsic rewards before being added to
              extrinsic ones.
           lr: The learning rate for the curiosity-specific
              optimizer, optimizing feature-, inverse-, and forward nets.
           sub_exploration: The config dict for
              the underlying Exploration to use (e.g. epsilon-greedy for
              DQN). If None, uses the FromSpecDict provided in the Policy's
              default config.
      """
      if not isinstance(action_space, (Discrete, MultiDiscrete)):
          raise ValueError(
              "Only (Multi)Discrete action spaces supported for Curiosity so far!"
          )
  
      super().__init__(
          action_space, model=model, framework=framework, **kwargs
      )
  
      if self.policy_config["num_workers"] != 0:
          raise ValueError(
              "Curiosity exploration currently does not support parallelism."
              " `num_workers` must be 0!"
          )
  
      self.feature_dim = feature_dim
      if feature_net_config is None:
          feature_net_config = self.policy_config["model"].copy()
      self.feature_net_config = feature_net_config
      self.inverse_net_hiddens = inverse_net_hiddens
      self.inverse_net_activation = inverse_net_activation
      self.forward_net_hiddens = forward_net_hiddens
      self.forward_net_activation = forward_net_activation
  
      self.action_dim = (
          self.action_space.n
          if isinstance(self.action_space, Discrete)
          else np.sum(self.action_space.nvec)
      )
  
      self.beta = beta
      self.eta = eta
      self.lr = lr
      # TODO: (sven) if sub_exploration is None, use Algorithm's default
      #  Exploration config.
      if sub_exploration is None:
          raise NotImplementedError
      self.sub_exploration = sub_exploration
  
      # Creates modules/layers inside the actual ModelV2.
      self._curiosity_feature_net = ModelCatalog.get_model_v2(
          self.model.obs_space,
          self.action_space,
          self.feature_dim,
          model_config=self.feature_net_config,
          framework=self.framework,
          name="feature_net",
      )
  
      self._curiosity_inverse_fcnet = self._create_fc_net(
          [2 * self.feature_dim]
          + list(self.inverse_net_hiddens)
          + [self.action_dim],
          self.inverse_net_activation,
          name="inverse_net",
      )
  
      self._curiosity_forward_fcnet = self._create_fc_net(
          [self.feature_dim + self.action_dim]
          + list(self.forward_net_hiddens)
          + [self.feature_dim],
          self.forward_net_activation,
          name="forward_net",
      )
  
      # This is only used to select the correct action
      self.exploration_submodule = from_config(
          cls=Exploration,
          config=self.sub_exploration,
          action_space=self.action_space,
          framework=self.framework,
          policy_config=self.policy_config,
          model=self.model,
          num_workers=self.num_workers,
          worker_index=self.worker_index,
      )
  
imports: from gymnasium.spaces import Discrete, MultiDiscrete, Space
  import numpy
  from typing import Optional, Tuple, Union
  from ray.rllib.models.action_dist import ActionDistribution
  from ray.rllib.models.catalog import ModelCatalog
  from ray.rllib.models.modelv2 import ModelV2
  from ray.rllib.models.tf.tf_action_dist import Categorical, MultiCategorical
  from ray.rllib.models.torch.misc import SlimFC
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical, TorchMultiCategorical
  from ray.rllib.models.utils import get_activation_fn
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils import NullContextManager
  from ray.rllib.utils.annotations import override, PublicAPI
  from ray.rllib.utils.exploration.exploration import Exploration
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.from_config import from_config
  from ray.rllib.utils.tf_utils import get_placeholder, one_hot
  from ray.rllib.utils.torch_utils import one_hot
  from ray.rllib.utils.typing import FromConfigSpec, ModelConfigDict, TensorType
label_desc: Use image node_id 1 to create a new Curiosity object from inherited base classes: Exploration with example: obj = Curiosity(action_space)