docstr: Test whether PPO can be built with all frameworks.
docstr_tok: ["Test","whether","PPO","can","be","built","with","all","frameworks","."]
code: def test_ppo_compilation_and_schedule_mixins(self):
      """Test whether PPO can be built with all frameworks."""
  
      # Build a PPOConfig object.
      config = (
          ppo.PPOConfig()
          # TODO (Kourosh): Enable when the scheduler is supported in the new
          #  Learner API stack.
          .experimental(_enable_new_api_stack=False)
          .training(
              # Setup lr schedule for testing.
              lr_schedule=[[0, 5e-5], [256, 0.0]],
              # Set entropy_coeff to a faulty value to proof that it'll get
              # overridden by the schedule below (which is expected).
              entropy_coeff=100.0,
              entropy_coeff_schedule=[[0, 0.1], [512, 0.0]],
              train_batch_size=256,
              sgd_minibatch_size=128,
              num_sgd_iter=2,
              model=dict(
                  # Settings in case we use an LSTM.
                  lstm_cell_size=10,
                  max_seq_len=20,
              ),
          )
          .rollouts(
              num_rollout_workers=1,
              # Test with compression.
              compress_observations=True,
          )
          .callbacks(MyCallbacks)
      )  # For checking lr-schedule correctness.
  
      num_iterations = 2
  
      for fw in framework_iterator(config):
          for env in ["FrozenLake-v1", "ALE/MsPacman-v5"]:
              print("Env={}".format(env))
              for lstm in [False, True]:
                  print("LSTM={}".format(lstm))
                  config.training(
                      model=dict(
                          use_lstm=lstm,
                          lstm_use_prev_action=lstm,
                          lstm_use_prev_reward=lstm,
                      )
                  )
  
                  algo = config.build(env=env)
                  policy = algo.get_policy()
                  entropy_coeff = algo.get_policy().entropy_coeff
                  lr = policy.cur_lr
                  if fw == "tf":
                      entropy_coeff, lr = policy.get_session().run(
                          [entropy_coeff, lr]
                      )
                  check(entropy_coeff, 0.1)
                  check(lr, config.lr)
  
                  for i in range(num_iterations):
                      results = algo.train()
                      print(results)
                      check_train_results(results)
                      # 2 sgd iters per update, 2 minibatches per trainbatch -> 4x
                      # avg(0.0, 1.0, 2.0, 3.0) -> 1.5
                      off_policy_ness = check_off_policyness(
                          results, lower_limit=1.5, upper_limit=1.5
                      )
                      print(f"off-policy'ness={off_policy_ness}")
  
                  check_compute_single_action(
                      algo,
                      include_prev_action_reward=True,
                      include_state=lstm,
                  )
                  algo.stop()
  
imports: import unittest
  import numpy
  import ray
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  import ray.rllib.algorithms.ppo
  from ray.rllib.algorithms.ppo.ppo_tf_policy import PPOTF2Policy
  from ray.rllib.algorithms.ppo.ppo_torch_policy import PPOTorchPolicy
  from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch, Postprocessing
  from ray.rllib.models.tf.tf_action_dist import Categorical
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, LEARNER_STATS_KEY
  from ray.rllib.utils.numpy import fc
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_off_policyness, check_train_results, framework_iterator, check_inference_w_connectors
label_desc: Use image node_id 4 for calling the TestPPO obj's underlying member method code with example usage: obj.test_ppo_compilation_and_schedule_mixins() without return types