docstr: Tests the base grad clipping logic in `postprocess_gradients()`.
docstr_tok: ["Tests","the","base","grad","clipping","logic","in","`","postprocess_gradients","(",")","`","."]
code: def test_postprocess_gradients(self):
      """Tests the base grad clipping logic in `postprocess_gradients()`."""
  
      # Clip by value only.
      config = BaseTestingAlgorithmConfig().training(
          lr=0.0003, grad_clip=0.75, grad_clip_by="value"
      )
  
      for fw in framework_iterator(config, frameworks=("torch", "tf2")):
          learner = config.build_learner(env=self.ENV)
          # Pretend our computed gradients are our weights + 1.0.
          grads = {
              learner.get_param_ref(v): v + 1.0
              for v in learner.get_parameters(
                  learner.module[DEFAULT_POLICY_ID]
              )
          }
          # Call the learner's postprocessing method.
          processed_grads = list(
              learner.postprocess_gradients(grads).values()
          )
          # Check clipped gradients.
          # No single gradient must be larger than 0.1 or smaller than -0.1:
          self.assertTrue(
              all(
                  np.max(grad) <= config.grad_clip
                  and np.min(grad) >= -config.grad_clip
                  for grad in convert_to_numpy(processed_grads)
              )
          )
  
          # Clip by norm.
          config.grad_clip = 1.0
          config.grad_clip_by = "norm"
          learner = config.build_learner(env=self.ENV)
          # Pretend our computed gradients are our weights + 1.0.
          grads = {
              learner.get_param_ref(v): v + 1.0
              for v in learner.get_parameters(
                  learner.module[DEFAULT_POLICY_ID]
              )
          }
          # Call the learner's postprocessing method.
          processed_grads = list(
              learner.postprocess_gradients(grads).values()
          )
          # Check clipped gradients.
          for proc_grad, grad in zip(
              convert_to_numpy(processed_grads),
              convert_to_numpy(list(grads.values())),
          ):
              l2_norm = np.sqrt(np.sum(grad**2.0))
              if l2_norm > config.grad_clip:
                  check(proc_grad, grad * (config.grad_clip / l2_norm))
  
          # Clip by global norm.
          config.grad_clip = 5.0
          config.grad_clip_by = "global_norm"
          learner = config.build_learner(env=self.ENV)
          # Pretend our computed gradients are our weights + 1.0.
          grads = {
              learner.get_param_ref(v): v + 1.0
              for v in learner.get_parameters(
                  learner.module[DEFAULT_POLICY_ID]
              )
          }
          # Call the learner's postprocessing method.
          processed_grads = list(
              learner.postprocess_gradients(grads).values()
          )
          # Check clipped gradients.
          global_norm = np.sqrt(
              np.sum(
                  np.sum(grad**2.0)
                  for grad in convert_to_numpy(list(grads.values()))
              )
          )
          if global_norm > config.grad_clip:
              for proc_grad, grad in zip(
                  convert_to_numpy(processed_grads),
                  grads.values(),
              ):
                  check(
                      proc_grad, grad * (config.grad_clip / global_norm)
                  )
  
imports: import gymnasium
  import numpy
  import tempfile
  import unittest
  import ray
  from ray.rllib.core.learner.learner import Learner
  from ray.rllib.core.testing.testing_learner import BaseTestingAlgorithmConfig
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
  from ray.rllib.utils.numpy import convert_to_numpy
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.test_utils import check, framework_iterator, get_cartpole_dataset_reader
  from ray.rllib.utils.metrics import ALL_MODULES
label_desc: Use image node_id 5 for calling the TestLearner obj's underlying member method code with example usage: obj.test_postprocess_gradients() without return types