docstr: Tests, whether PPO runs with different exploration setups.
docstr_tok: ["Tests",",","whether","PPO","runs","with","different","exploration","setups","."]
code: def test_ppo_exploration_setup(self):
      """Tests, whether PPO runs with different exploration setups."""
      config = (
          ppo.PPOConfig()
          .experimental(_enable_new_api_stack=True)
          .environment(
              "FrozenLake-v1",
              env_config={"is_slippery": False, "map_name": "4x4"},
          )
          .rollouts(
              # Run locally.
              num_rollout_workers=0,
          )
      )
      obs = np.array(0)
  
      # Test against all frameworks.
      for fw in framework_iterator(config):
          # Default Agent should be setup with StochasticSampling.
          algo = config.build()
          # explore=False, always expect the same (deterministic) action.
          a_ = algo.compute_single_action(
              obs,
              explore=False,
              prev_action=np.array(2),
              prev_reward=np.array(1.0),
          )
  
          # Test whether this is really the argmax action over the logits.
          # TODO (Kourosh): Only meaningful in the ModelV2 stack.
          config.validate()
          if not config._enable_new_api_stack and fw != "tf":
              last_out = algo.get_policy().model.last_output()
              if fw == "torch":
                  check(
                      a_,
                      np.argmax(last_out.detach().cpu().numpy(), 1)[0],
                  )
              else:
                  check(a_, np.argmax(last_out.numpy(), 1)[0])
  
          for _ in range(50):
              a = algo.compute_single_action(
                  obs,
                  explore=False,
                  prev_action=np.array(2),
                  prev_reward=np.array(1.0),
              )
              check(a, a_)
  
          # With explore=True (default), expect stochastic actions.
          actions = []
          for _ in range(300):
              actions.append(
                  algo.compute_single_action(
                      obs,
                      prev_action=np.array(2),
                      prev_reward=np.array(1.0),
                  )
              )
          check(np.mean(actions), 1.5, atol=0.2)
          algo.stop()
  
imports: import unittest
  import numpy
  import ray
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  import ray.rllib.algorithms.ppo
  from ray.rllib.algorithms.ppo.ppo_tf_policy import PPOTF2Policy
  from ray.rllib.algorithms.ppo.ppo_torch_policy import PPOTorchPolicy
  from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch, Postprocessing
  from ray.rllib.models.tf.tf_action_dist import Categorical
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, LEARNER_STATS_KEY
  from ray.rllib.utils.numpy import fc
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_off_policyness, check_train_results, framework_iterator, check_inference_w_connectors
label_desc: Use image node_id 5 for calling the TestPPO obj's underlying member method code with example usage: obj.test_ppo_exploration_setup() without return types