docstr: Tests the free log std option works.
docstr_tok: ["Tests","the","free","log","std","option","works","."]
code: def test_ppo_free_log_std_with_rl_modules(self):
      """Tests the free log std option works."""
      config = (
          ppo.PPOConfig()
          .experimental(_enable_new_api_stack=True)
          .environment("Pendulum-v1")
          .rollouts(
              num_rollout_workers=1,
          )
          .training(
              gamma=0.99,
              model=dict(
                  fcnet_hiddens=[10],
                  fcnet_activation="linear",
                  free_log_std=True,
                  vf_share_layers=True,
              ),
          )
      )
  
      for fw in framework_iterator(config, frameworks=("torch", "tf2")):
          algo = config.build()
          policy = algo.get_policy()
          learner = algo.learner_group._learner
          module = learner.module[DEFAULT_POLICY_ID]
  
          # Check the free log std var is created.
          if fw == "torch":
              matching = [
                  v
                  for (n, v) in module.named_parameters()
                  if "log_std" in n
              ]
          else:
              matching = [
                  v
                  for v in module.trainable_variables
                  if "log_std" in str(v)
              ]
          assert len(matching) == 1, matching
          log_std_var = matching[0]
  
          def get_value():
              if fw == "torch":
                  return log_std_var.detach().cpu().numpy()[0]
              else:
                  return log_std_var.numpy()[0]
  
          # Check the variable is initially zero.
          init_std = get_value()
          assert init_std == 0.0, init_std
          batch = compute_gae_for_sample_batch(
              policy, PENDULUM_FAKE_BATCH.copy()
          )
          batch = policy._lazy_tensor_dict(batch)
          algo.learner_group.update_from_batch(
              batch=batch.as_multi_agent()
          )
  
          # Check the variable is updated.
          post_std = get_value()
          assert post_std != 0.0, post_std
          algo.stop()
  
imports: import unittest
  import numpy
  import ray
  import ray.rllib.algorithms.ppo
  from ray.rllib.algorithms.ppo.ppo import LEARNER_RESULTS_CURR_ENTROPY_COEFF_KEY
  from ray.rllib.algorithms.callbacks import DefaultCallbacks
  from ray.rllib.algorithms.ppo.tests.test_ppo import PENDULUM_FAKE_BATCH
  from ray.rllib.core.learner.learner import LEARNER_RESULTS_CURR_LR_KEY
  from ray.rllib.evaluation.postprocessing import compute_gae_for_sample_batch
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
  from ray.rllib.utils.metrics.learner_info import LEARNER_INFO
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
label_desc: Use image node_id 5 for calling the TestPPO obj's underlying member method code with example usage: obj.test_ppo_free_log_std_with_rl_modules() and returns: log_std_var, log_std_var