docstr: Tests the free log std option works.
  
  This test is overfitted to the old ModelV2 stack (e.g.
  policy.model.trainable_variables is not callable in the new stack)
  # TODO (Kourosh) we should create a new test for the new RLModule stack.
docstr_tok: ["Tests","the","free","log","std","option","works",".","This","test","is","overfitted","to","the","old","ModelV2","stack","(","e.g",".","policy.model.trainable_variables","is","not","callable","in","the","new","stack",")","#","TODO","(","Kourosh",")","we","should","create","a","new","test","for","the","new","RLModule","stack","."]
code: def test_ppo_free_log_std(self):
      """Tests the free log std option works.
  
      This test is overfitted to the old ModelV2 stack (e.g.
      policy.model.trainable_variables is not callable in the new stack)
      # TODO (Kourosh) we should create a new test for the new RLModule stack.
      """
  
      config = (
          ppo.PPOConfig()
          # TODO (Kourosh): Enable when free log std is supported in the new
          #  Learner API stack.
          .experimental(_enable_new_api_stack=False)
          .environment("CartPole-v1")
          .rollouts(
              num_rollout_workers=0,
          )
          .training(
              gamma=0.99,
              model=dict(
                  fcnet_hiddens=[10],
                  fcnet_activation="linear",
                  free_log_std=True,
                  vf_share_layers=True,
              ),
          )
      )
  
      for fw, sess in framework_iterator(config, session=True):
          algo = config.build()
          policy = algo.get_policy()
  
          # Check the free log std var is created.
          if fw == "torch":
              matching = [
                  v
                  for (n, v) in policy.model.named_parameters()
                  if "log_std" in n
              ]
          else:
              matching = [
                  v
                  for v in policy.model.trainable_variables()
                  if "log_std" in str(v)
              ]
          assert len(matching) == 1, matching
          log_std_var = matching[0]
  
          # linter yells at you if you don't pass in the parameters.
          # reason: https://docs.python-guide.org/writing/gotchas/
          # #late-binding-closures
          def get_value(fw=fw, policy=policy, log_std_var=log_std_var):
              if fw == "tf":
                  return policy.get_session().run(log_std_var)[0]
              elif fw == "torch":
                  return log_std_var.detach().cpu().numpy()[0]
              else:
                  return log_std_var.numpy()[0]
  
          # Check the variable is initially zero.
          init_std = get_value()
          assert init_std == 0.0, init_std
          batch = compute_gae_for_sample_batch(
              policy, CARTPOLE_FAKE_BATCH.copy()
          )
          if fw == "torch":
              batch = policy._lazy_tensor_dict(batch)
          policy.learn_on_batch(batch)
  
          # Check the variable is updated.
          post_std = get_value()
          assert post_std != 0.0, post_std
          algo.stop()
  
imports: import unittest
  import numpy
  import ray
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  import ray.rllib.algorithms.ppo
  from ray.rllib.algorithms.ppo.ppo_tf_policy import PPOTF2Policy
  from ray.rllib.algorithms.ppo.ppo_torch_policy import PPOTorchPolicy
  from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch, Postprocessing
  from ray.rllib.models.tf.tf_action_dist import Categorical
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.models.torch.torch_action_dist import TorchCategorical
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch
  from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, LEARNER_STATS_KEY
  from ray.rllib.utils.numpy import fc
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_off_policyness, check_train_results, framework_iterator, check_inference_w_connectors
label_desc: Use image node_id 6 for calling the TestPPO obj's underlying member method code with example usage: obj.test_ppo_free_log_std() and returns: policy, log_std_var, log_std_var