class_docstr: An exploration that changes a Model's parameters.
  
  Implemented based on:
  [1] https://openai.com/research/better-exploration-with-parameter-noise
  [2] https://arxiv.org/pdf/1706.01905.pdf
  
  At the beginning of an episode, Gaussian noise is added to all weights
  of the model. At the end of the episode, the noise is undone and an action
  diff (pi-delta) is calculated, from which we determine the changes in the
  noise's stddev for the next episode.
class_docstr_tok: ["An","exploration","that","changes","a","Model","'s","parameters",".","Implemented","based","on",":","[","1","]","https",":","\/\/openai.com\/research\/better-exploration-with-parameter-noise","[","2","]","https",":","\/\/arxiv.org\/pdf\/1706.01905.pdf","At","the","beginning","of","an","episode",",","Gaussian","noise","is","added","to","all","weights","of","the","model",".","At","the","end","of","the","episode",",","the","noise","is","undone","and","an","action","diff","(","pi-delta",")","is","calculated",",","from","which","we","determine","the","changes","in","the","noise","'s","stddev","for","the","next","episode","."]
docstr: Initializes a ParameterNoise Exploration object.
  
  Args:
      initial_stddev: The initial stddev to use for the noise.
      random_timesteps: The number of timesteps to act completely
          randomly (see [1]).
      sub_exploration: Optional sub-exploration config.
          None for auto-detection/setup.
docstr_tok: ["Initializes","a","ParameterNoise","Exploration","object",".","Args",":","initial_stddev",":","The","initial","stddev","to","use","for","the","noise",".","random_timesteps",":","The","number","of","timesteps","to","act","completely","randomly","(","see","[","1","]",")",".","sub_exploration",":","Optional","sub-exploration","config",".","None","for","auto-detection\/setup","."]
code: def __init__(
      self,
      action_space,
      *,
      framework: str,
      policy_config: dict,
      model: ModelV2,
      initial_stddev: float = 1.0,
      random_timesteps: int = 10000,
      sub_exploration: Optional[dict] = None,
      **kwargs,
  ):
      """Initializes a ParameterNoise Exploration object.
  
      Args:
          initial_stddev: The initial stddev to use for the noise.
          random_timesteps: The number of timesteps to act completely
              randomly (see [1]).
          sub_exploration: Optional sub-exploration config.
              None for auto-detection/setup.
      """
      assert framework is not None
      super().__init__(
          action_space,
          policy_config=policy_config,
          model=model,
          framework=framework,
          **kwargs,
      )
  
      self.stddev = get_variable(
          initial_stddev, framework=self.framework, tf_name="stddev"
      )
      self.stddev_val = initial_stddev  # Out-of-graph tf value holder.
  
      # The weight variables of the Model where noise should be applied to.
      # This excludes any variable, whose name contains "LayerNorm" (those
      # are BatchNormalization layers, which should not be perturbed).
      self.model_variables = [
          v
          for k, v in self.model.trainable_variables(
              as_dict=True
          ).items()
          if "LayerNorm" not in k
      ]
      # Our noise to be added to the weights. Each item in `self.noise`
      # corresponds to one Model variable and holding the Gaussian noise to
      # be added to that variable (weight).
      self.noise = []
      for var in self.model_variables:
          name_ = var.name.split(":")[0] + "_noisy" if var.name else ""
          self.noise.append(
              get_variable(
                  np.zeros(var.shape, dtype=np.float32),
                  framework=self.framework,
                  tf_name=name_,
                  torch_tensor=True,
                  device=self.device,
              )
          )
  
      # tf-specific ops to sample, assign and remove noise.
      if self.framework == "tf" and not tf.executing_eagerly():
          self.tf_sample_new_noise_op = self._tf_sample_new_noise_op()
          self.tf_add_stored_noise_op = self._tf_add_stored_noise_op()
          self.tf_remove_noise_op = self._tf_remove_noise_op()
          # Create convenience sample+add op for tf.
          with tf1.control_dependencies([self.tf_sample_new_noise_op]):
              add_op = self._tf_add_stored_noise_op()
          with tf1.control_dependencies([add_op]):
              self.tf_sample_new_noise_and_add_op = tf.no_op()
  
      # Whether the Model's weights currently have noise added or not.
      self.weights_are_currently_noisy = False
  
      # Auto-detection of underlying exploration functionality.
      if sub_exploration is None:
          # For discrete action spaces, use an underlying EpsilonGreedy with
          # a special schedule.
          if isinstance(self.action_space, Discrete):
              sub_exploration = {
                  "type": "EpsilonGreedy",
                  "epsilon_schedule": {
                      "type": "PiecewiseSchedule",
                      # Step function (see [2]).
                      "endpoints": [
                          (0, 1.0),
                          (random_timesteps + 1, 1.0),
                          (random_timesteps + 2, 0.01),
                      ],
                      "outside_value": 0.01,
                  },
              }
          elif isinstance(self.action_space, Box):
              sub_exploration = {
                  "type": "OrnsteinUhlenbeckNoise",
                  "random_timesteps": random_timesteps,
              }
          # TODO(sven): Implement for any action space.
          else:
              raise NotImplementedError
  
      self.sub_exploration = from_config(
          Exploration,
          sub_exploration,
          framework=self.framework,
          action_space=self.action_space,
          policy_config=self.policy_config,
          model=self.model,
          **kwargs,
      )
  
      # Whether we need to call `self._delayed_on_episode_start` before
      # the forward pass.
      self.episode_started = False
  
imports: from gymnasium.spaces import Box, Discrete
  import numpy
  from typing import Optional, TYPE_CHECKING, Union
  from ray.rllib.env.base_env import BaseEnv
  from ray.rllib.models.action_dist import ActionDistribution
  from ray.rllib.models.modelv2 import ModelV2
  from ray.rllib.models.tf.tf_action_dist import Categorical, Deterministic
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical, TorchDeterministic
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import override, PublicAPI
  from ray.rllib.utils.exploration.exploration import Exploration
  from ray.rllib.utils.framework import get_variable, try_import_tf, try_import_torch
  from ray.rllib.utils.from_config import from_config
  from ray.rllib.utils.numpy import softmax, SMALL_NUMBER
  from ray.rllib.utils.typing import TensorType
label_desc: Use image node_id 1 to create a new ParameterNoise object from inherited base classes: Exploration with example: obj = ParameterNoise(action_space)