class_docstr: Random Encoder for Efficient Exploration.
  
  Implementation of:
  [1] State entropy maximization with random encoders for efficient
  exploration. Seo, Chen, Shin, Lee, Abbeel, & Lee, (2021).
  arXiv preprint arXiv:2102.09430.
  
  Estimates state entropy using a particle-based k-nearest neighbors (k-NN)
  estimator in the latent space. The state's latent representation is
  calculated using an encoder with randomly initialized parameters.
  
  The entropy of a state is considered as intrinsic reward and added to the
  environment's extrinsic reward for policy optimization.
  Entropy is calculated per batch, it does not take the distribution of
  the entire replay buffer into consideration.
class_docstr_tok: ["Random","Encoder","for","Efficient","Exploration",".","Implementation","of",":","[","1","]","State","entropy","maximization","with","random","encoders","for","efficient","exploration",".","Seo",",","Chen",",","Shin",",","Lee",",","Abbeel",",","&","Lee",",","(","2021",")",".","arXiv","preprint","arXiv:2102.09430",".","Estimates","state","entropy","using","a","particle-based","k-nearest","neighbors","(","k-NN",")","estimator","in","the","latent","space",".","The","state","'s","latent","representation","is","calculated","using","an","encoder","with","randomly","initialized","parameters",".","The","entropy","of","a","state","is","considered","as","intrinsic","reward","and","added","to","the","environment","'s","extrinsic","reward","for","policy","optimization",".","Entropy","is","calculated","per","batch",",","it","does","not","take","the","distribution","of","the","entire","replay","buffer","into","consideration","."]
docstr: Initialize RE3.
  
  Args:
      action_space: The action space in which to explore.
      framework: Supports "tf", this implementation does not
          support torch.
      model: The policy's model.
      embeds_dim: The dimensionality of the observation embedding
          vectors in latent space.
      encoder_net_config: Optional model
          configuration for the encoder network, producing embedding
          vectors from observations. This can be used to configure
          fcnet- or conv_net setups to properly process any
          observation space.
      beta: Hyperparameter to choose between exploration and
          exploitation.
      beta_schedule: Schedule to use for beta decay, one of
          "constant" or "linear_decay".
      rho: Beta decay factor, used for on-policy algorithm.
      k_nn: Number of neighbours to set for K-NN entropy
          estimation.
      random_timesteps: The number of timesteps to act completely
          randomly (see [1]).
      sub_exploration: The config dict for the underlying Exploration
          to use (e.g. epsilon-greedy for DQN). If None, uses the
          FromSpecDict provided in the Policy's default config.
  
  Raises:
      ValueError: If the input framework is Torch.
docstr_tok: ["Initialize","RE3",".","Args",":","action_space",":","The","action","space","in","which","to","explore",".","framework",":","Supports","``","tf","''",",","this","implementation","does","not","support","torch",".","model",":","The","policy","'s","model",".","embeds_dim",":","The","dimensionality","of","the","observation","embedding","vectors","in","latent","space",".","encoder_net_config",":","Optional","model","configuration","for","the","encoder","network",",","producing","embedding","vectors","from","observations",".","This","can","be","used","to","configure","fcnet-","or","conv_net","setups","to","properly","process","any","observation","space",".","beta",":","Hyperparameter","to","choose","between","exploration","and","exploitation",".","beta_schedule",":","Schedule","to","use","for","beta","decay",",","one","of","``","constant","''","or","``","linear_decay","''",".","rho",":","Beta","decay","factor",",","used","for","on-policy","algorithm",".","k_nn",":","Number","of","neighbours","to","set","for","K-NN","entropy","estimation",".","random_timesteps",":","The","number","of","timesteps","to","act","completely","randomly","(","see","[","1","]",")",".","sub_exploration",":","The","config","dict","for","the","underlying","Exploration","to","use","(","e.g",".","epsilon-greedy","for","DQN",")",".","If","None",",","uses","the","FromSpecDict","provided","in","the","Policy","'s","default","config",".","Raises",":","ValueError",":","If","the","input","framework","is","Torch","."]
code: def __init__(
      self,
      action_space: Space,
      *,
      framework: str,
      model: ModelV2,
      embeds_dim: int = 128,
      encoder_net_config: Optional[ModelConfigDict] = None,
      beta: float = 0.2,
      beta_schedule: str = "constant",
      rho: float = 0.1,
      k_nn: int = 50,
      random_timesteps: int = 10000,
      sub_exploration: Optional[FromConfigSpec] = None,
      **kwargs,
  ):
      """Initialize RE3.
  
      Args:
          action_space: The action space in which to explore.
          framework: Supports "tf", this implementation does not
              support torch.
          model: The policy's model.
          embeds_dim: The dimensionality of the observation embedding
              vectors in latent space.
          encoder_net_config: Optional model
              configuration for the encoder network, producing embedding
              vectors from observations. This can be used to configure
              fcnet- or conv_net setups to properly process any
              observation space.
          beta: Hyperparameter to choose between exploration and
              exploitation.
          beta_schedule: Schedule to use for beta decay, one of
              "constant" or "linear_decay".
          rho: Beta decay factor, used for on-policy algorithm.
          k_nn: Number of neighbours to set for K-NN entropy
              estimation.
          random_timesteps: The number of timesteps to act completely
              randomly (see [1]).
          sub_exploration: The config dict for the underlying Exploration
              to use (e.g. epsilon-greedy for DQN). If None, uses the
              FromSpecDict provided in the Policy's default config.
  
      Raises:
          ValueError: If the input framework is Torch.
      """
      # TODO(gjoliver): Add supports for Pytorch.
      if framework == "torch":
          raise ValueError(
              "This RE3 implementation does not support Torch."
          )
      super().__init__(
          action_space, model=model, framework=framework, **kwargs
      )
  
      self.beta = beta
      self.rho = rho
      self.k_nn = k_nn
      self.embeds_dim = embeds_dim
      if encoder_net_config is None:
          encoder_net_config = self.policy_config["model"].copy()
      self.encoder_net_config = encoder_net_config
  
      # Auto-detection of underlying exploration functionality.
      if sub_exploration is None:
          # For discrete action spaces, use an underlying EpsilonGreedy with
          # a special schedule.
          if isinstance(self.action_space, Discrete):
              sub_exploration = {
                  "type": "EpsilonGreedy",
                  "epsilon_schedule": {
                      "type": "PiecewiseSchedule",
                      # Step function (see [2]).
                      "endpoints": [
                          (0, 1.0),
                          (random_timesteps + 1, 1.0),
                          (random_timesteps + 2, 0.01),
                      ],
                      "outside_value": 0.01,
                  },
              }
          elif isinstance(self.action_space, Box):
              sub_exploration = {
                  "type": "OrnsteinUhlenbeckNoise",
                  "random_timesteps": random_timesteps,
              }
          else:
              raise NotImplementedError
  
      self.sub_exploration = sub_exploration
  
      # Creates ModelV2 embedding module / layers.
      self._encoder_net = ModelCatalog.get_model_v2(
          self.model.obs_space,
          self.action_space,
          self.embeds_dim,
          model_config=self.encoder_net_config,
          framework=self.framework,
          name="encoder_net",
      )
      if self.framework == "tf":
          self._obs_ph = get_placeholder(
              space=self.model.obs_space, name="_encoder_obs"
          )
          self._obs_embeds = tf.stop_gradient(
              self._encoder_net({SampleBatch.OBS: self._obs_ph})[0]
          )
  
      # This is only used to select the correct action
      self.exploration_submodule = from_config(
          cls=Exploration,
          config=self.sub_exploration,
          action_space=self.action_space,
          framework=self.framework,
          policy_config=self.policy_config,
          model=self.model,
          num_workers=self.num_workers,
          worker_index=self.worker_index,
      )
  
imports: from gymnasium.spaces import Box, Discrete, Space
  import numpy
  from typing import List, Optional, Union
  from ray.rllib.models.action_dist import ActionDistribution
  from ray.rllib.models.catalog import ModelCatalog
  from ray.rllib.models.modelv2 import ModelV2
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import override, PublicAPI
  from ray.rllib.utils.exploration.exploration import Exploration
  from ray.rllib.utils.framework import try_import_tf
  from ray.rllib.utils.from_config import from_config
  from ray.rllib.utils.tf_utils import get_placeholder
  from ray.rllib.utils.typing import FromConfigSpec, ModelConfigDict, TensorType
label_desc: Use image node_id 1 to create a new RE3 object from inherited base classes: Exploration with example: obj = RE3(action_space)