docstr: Tests, whether a DQN Agent outputs exploration/softmaxed actions.
docstr_tok: ["Tests",",","whether","a","DQN","Agent","outputs","exploration\/softmaxed","actions","."]
code: def test_dqn_exploration_and_soft_q_config(self):
      """Tests, whether a DQN Agent outputs exploration/softmaxed actions."""
      config = (
          dqn.dqn.DQNConfig()
          .environment("FrozenLake-v1")
          .rollouts(num_rollout_workers=0)
          .environment(
              env_config={"is_slippery": False, "map_name": "4x4"}
          )
      ).training(num_steps_sampled_before_learning_starts=0)
  
      obs = np.array(0)
  
      # Test against all frameworks.
      for _ in framework_iterator(config):
          # Default EpsilonGreedy setup.
          algo = config.build()
          # Setting explore=False should always return the same action.
          a_ = algo.compute_single_action(obs, explore=False)
          for _ in range(50):
              a = algo.compute_single_action(obs, explore=False)
              check(a, a_)
          # explore=None (default: explore) should return different actions.
          actions = []
          for _ in range(50):
              actions.append(algo.compute_single_action(obs))
          check(np.std(actions), 0.0, false=True)
          algo.stop()
  
          # Low softmax temperature. Behaves like argmax
          # (but no epsilon exploration).
          config.exploration(
              exploration_config={
                  "type": "SoftQ",
                  "temperature": 0.000001,
              }
          )
          algo = config.build()
          # Due to the low temp, always expect the same action.
          actions = [algo.compute_single_action(obs)]
          for _ in range(50):
              actions.append(algo.compute_single_action(obs))
          check(np.std(actions), 0.0, decimals=3)
          algo.stop()
  
          # Higher softmax temperature.
          config.exploration_config["temperature"] = 1.0
          algo = config.build()
  
          # Even with the higher temperature, if we set explore=False, we
          # should expect the same actions always.
          a_ = algo.compute_single_action(obs, explore=False)
          for _ in range(50):
              a = algo.compute_single_action(obs, explore=False)
              check(a, a_)
  
          # Due to the higher temp, expect different actions avg'ing
          # around 1.5.
          actions = []
          for _ in range(300):
              actions.append(algo.compute_single_action(obs))
          check(np.std(actions), 0.0, false=True)
          algo.stop()
  
          # With Random exploration.
          config.exploration(
              exploration_config={"type": "Random"}, explore=True
          )
          algo = config.build()
          actions = []
          for _ in range(300):
              actions.append(algo.compute_single_action(obs))
          check(np.std(actions), 0.0, false=True)
          algo.stop()
  
file_path: ray/rllib/algorithms/dqn/tests/test_dqn.py
imports: from copy import deepcopy
  import numpy
  import unittest
  import ray
  import ray.rllib.algorithms.dqn
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
label_desc: Use image node_id 5 for calling the TestDQN obj's underlying member method code with example usage: obj.test_dqn_exploration_and_soft_q_config() without return types