class_docstr: Implements torch-specific PPO loss logic on top of PPOLearner.
  
  This class implements the ppo loss under `self.compute_loss_for_module()`.
class_docstr_tok: ["Implements","torch-specific","PPO","loss","logic","on","top","of","PPOLearner",".","This","class","implements","the","ppo","loss","under","`","self.compute_loss_for_module","(",")","`","."]
code: def additional_update_for_module(
      self,
      *,
      module_id: ModuleID,
      config: PPOConfig,
      timestep: int,
      sampled_kl_values: dict,
  ) -> Dict[str, Any]:
      assert sampled_kl_values, "Sampled KL values are empty."
  
      results = super().additional_update_for_module(
          module_id=module_id,
          config=config,
          timestep=timestep,
          sampled_kl_values=sampled_kl_values,
      )
  
      # Update KL coefficient.
      if config.use_kl_loss:
          sampled_kl = sampled_kl_values[module_id]
          curr_var = self.curr_kl_coeffs_per_module[module_id]
          if sampled_kl > 2.0 * config.kl_target:
              # TODO (Kourosh) why not 2?
              curr_var.data *= 1.5
          elif sampled_kl < 0.5 * config.kl_target:
              curr_var.data *= 0.5
          results.update(
              {LEARNER_RESULTS_CURR_KL_COEFF_KEY: curr_var.item()}
          )
  
      return results
  
file_path: ray/python/ray/rllib/algorithms/ppo/torch/ppo_torch_learner.py
imports: import logging
  from typing import Any, Dict
  from ray.rllib.algorithms.ppo.ppo import LEARNER_RESULTS_KL_KEY, LEARNER_RESULTS_CURR_KL_COEFF_KEY, LEARNER_RESULTS_VF_EXPLAINED_VAR_KEY, LEARNER_RESULTS_VF_LOSS_UNCLIPPED_KEY, PPOConfig
  from ray.rllib.algorithms.ppo.ppo_learner import PPOLearner
  from ray.rllib.utils.torch_utils import sequence_mask
  from ray.rllib.core.learner.learner import POLICY_LOSS_KEY, VF_LOSS_KEY, ENTROPY_KEY
  from ray.rllib.core.learner.torch.torch_learner import TorchLearner
  from ray.rllib.evaluation.postprocessing import Postprocessing
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.nested_dict import NestedDict
  from ray.rllib.utils.torch_utils import explained_variance
  from ray.rllib.utils.typing import ModuleID, TensorType
label_desc: Use image node_id 2 for calling the PPOTorchLearner obj's underlying member method code with example usage: obj.additional_update_for_module() and returns: results