class_docstr: An exploration that changes a Model's parameters.
  
  Implemented based on:
  [1] https://openai.com/research/better-exploration-with-parameter-noise
  [2] https://arxiv.org/pdf/1706.01905.pdf
  
  At the beginning of an episode, Gaussian noise is added to all weights
  of the model. At the end of the episode, the noise is undone and an action
  diff (pi-delta) is calculated, from which we determine the changes in the
  noise's stddev for the next episode.
class_docstr_tok: ["An","exploration","that","changes","a","Model","'s","parameters",".","Implemented","based","on",":","[","1","]","https",":","\/\/openai.com\/research\/better-exploration-with-parameter-noise","[","2","]","https",":","\/\/arxiv.org\/pdf\/1706.01905.pdf","At","the","beginning","of","an","episode",",","Gaussian","noise","is","added","to","all","weights","of","the","model",".","At","the","end","of","the","episode",",","the","noise","is","undone","and","an","action","diff","(","pi-delta",")","is","calculated",",","from","which","we","determine","the","changes","in","the","noise","'s","stddev","for","the","next","episode","."]
code: def postprocess_trajectory(
      self,
      policy: "Policy",
      sample_batch: SampleBatch,
      tf_sess: Optional["tf.Session"] = None,
  ):
      noisy_action_dist = noise_free_action_dist = None
      # Adjust the stddev depending on the action (pi)-distance.
      # Also see [1] for details.
      # TODO(sven): Find out whether this can be scrapped by simply using
      #  the `sample_batch` to get the noisy/noise-free action dist.
      _, _, fetches = policy.compute_actions_from_input_dict(
          input_dict=sample_batch,
          explore=self.weights_are_currently_noisy,
      )
  
      # Categorical case (e.g. DQN).
      if issubclass(policy.dist_class, (Categorical, TorchCategorical)):
          action_dist = softmax(fetches[SampleBatch.ACTION_DIST_INPUTS])
      # Deterministic (Gaussian actions, e.g. DDPG).
      elif issubclass(
          policy.dist_class, (Deterministic, TorchDeterministic)
      ):
          action_dist = fetches[SampleBatch.ACTION_DIST_INPUTS]
      else:
          raise NotImplementedError  # TODO(sven): Other action-dist cases.
  
      if self.weights_are_currently_noisy:
          noisy_action_dist = action_dist
      else:
          noise_free_action_dist = action_dist
  
      _, _, fetches = policy.compute_actions_from_input_dict(
          input_dict=sample_batch,
          explore=not self.weights_are_currently_noisy,
      )
  
      # Categorical case (e.g. DQN).
      if issubclass(policy.dist_class, (Categorical, TorchCategorical)):
          action_dist = softmax(fetches[SampleBatch.ACTION_DIST_INPUTS])
          # Deterministic (Gaussian actions, e.g. DDPG).
      elif issubclass(
          policy.dist_class, (Deterministic, TorchDeterministic)
      ):
          action_dist = fetches[SampleBatch.ACTION_DIST_INPUTS]
  
      if noisy_action_dist is None:
          noisy_action_dist = action_dist
      else:
          noise_free_action_dist = action_dist
  
      delta = distance = None
      # Categorical case (e.g. DQN).
      if issubclass(policy.dist_class, (Categorical, TorchCategorical)):
          # Calculate KL-divergence (DKL(clean||noisy)) according to [2].
          # TODO(sven): Allow KL-divergence to be calculated by our
          #  Distribution classes (don't support off-graph/numpy yet).
          distance = np.nanmean(
              np.sum(
                  noise_free_action_dist
                  * np.log(
                      noise_free_action_dist
                      / (noisy_action_dist + SMALL_NUMBER)
                  ),
                  1,
              )
          )
          current_epsilon = self.sub_exploration.get_state(
              sess=tf_sess
          )["cur_epsilon"]
          delta = -np.log(
              1
              - current_epsilon
              + current_epsilon / self.action_space.n
          )
      elif issubclass(
          policy.dist_class, (Deterministic, TorchDeterministic)
      ):
          # Calculate MSE between noisy and non-noisy output (see [2]).
          distance = np.sqrt(
              np.mean(
                  np.square(noise_free_action_dist - noisy_action_dist)
              )
          )
          current_scale = self.sub_exploration.get_state(sess=tf_sess)[
              "cur_scale"
          ]
          delta = (
              getattr(self.sub_exploration, "ou_sigma", 0.2)
              * current_scale
          )
  
      # Adjust stddev according to the calculated action-distance.
      if distance <= delta:
          self.stddev_val *= 1.01
      else:
          self.stddev_val /= 1.01
  
      # Update our state (self.stddev and self.stddev_val).
      self.set_state(self.get_state(), sess=tf_sess)
  
      return sample_batch
  
imports: from gymnasium.spaces import Box, Discrete
  import numpy
  from typing import Optional, TYPE_CHECKING, Union
  from ray.rllib.env.base_env import BaseEnv
  from ray.rllib.models.action_dist import ActionDistribution
  from ray.rllib.models.modelv2 import ModelV2
  from ray.rllib.models.tf.tf_action_dist import Categorical, Deterministic
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical, TorchDeterministic
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import override, PublicAPI
  from ray.rllib.utils.exploration.exploration import Exploration
  from ray.rllib.utils.framework import get_variable, try_import_tf, try_import_torch
  from ray.rllib.utils.from_config import from_config
  from ray.rllib.utils.numpy import softmax, SMALL_NUMBER
  from ray.rllib.utils.typing import TensorType
label_desc: Use image node_id 7 for calling the ParameterNoise obj's underlying member method code with example usage: obj.postprocess_trajectory(policy, sample_batch, tf_sess) and returns: sample_batch