class_docstr: Adds KL coeff updates via `additional_update_for_module()` to Impala logic.
  
  Framework-specific sub-classes must override `_update_module_target_networks()`
  and `_update_module_kl_coeff()`
class_docstr_tok: ["Adds","KL","coeff","updates","via","`","additional_update_for_module","(",")","`","to","Impala","logic",".","Framework-specific","sub-classes","must","override","`","_update_module_target_networks","(",")","`","and","`","_update_module_kl_coeff","(",")","`"]
docstr: Updates the target networks and KL loss coefficients (per module).
  
  Args:
      module_id:
docstr_tok: ["Updates","the","target","networks","and","KL","loss","coefficients","(","per","module",")",".","Args",":","module_id",":"]
code: def additional_update_for_module(
      self,
      *,
      module_id: ModuleID,
      config: APPOConfig,
      timestep: int,
      last_update: int,
      mean_kl_loss_per_module: dict,
      **kwargs,
  ) -> Dict[str, Any]:
      """Updates the target networks and KL loss coefficients (per module).
  
      Args:
          module_id:
      """
      # TODO (avnish) Using steps trained here instead of sampled ... I'm not sure
      #  why the other implementation uses sampled.
      #  The difference in steps sampled/trained is pretty
      #  much always going to be larger than self.config.num_sgd_iter *
      #  self.config.minibatch_buffer_size unless the number of steps collected
      #  is really small. The thing is that the default rollout fragment length
      #  is 50, so the minibatch buffer size * num_sgd_iter is going to be
      #  have to be 50 to even meet the threshold of having delayed target
      #  updates.
      #  We should instead have the target / kl threshold update be based off
      #  of the train_batch_size * some target update frequency * num_sgd_iter.
      results = super().additional_update_for_module(
          module_id=module_id, config=config, timestep=timestep
      )
  
      if (timestep - last_update) >= config.target_update_frequency:
          self._update_module_target_networks(module_id, config)
          results[NUM_TARGET_UPDATES] = 1
          results[LAST_TARGET_UPDATE_TS] = timestep
      else:
          results[NUM_TARGET_UPDATES] = 0
          results[LAST_TARGET_UPDATE_TS] = last_update
  
      if config.use_kl_loss and module_id in mean_kl_loss_per_module:
          results.update(
              self._update_module_kl_coeff(
                  module_id, config, mean_kl_loss_per_module[module_id]
              )
          )
  
      return results
  
file_path: ray/python/ray/rllib/algorithms/appo/appo_learner.py
imports: import abc
  from typing import Any, Dict
  from ray.rllib.algorithms.appo.appo import APPOConfig
  from ray.rllib.algorithms.impala.impala_learner import ImpalaLearner
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.lambda_defaultdict import LambdaDefaultDict
  from ray.rllib.utils.metrics import LAST_TARGET_UPDATE_TS, NUM_TARGET_UPDATES
  from ray.rllib.utils.schedules.scheduler import Scheduler
  from ray.rllib.utils.typing import ModuleID
label_desc: Use image node_id 3 for calling the AppoLearner obj's underlying member method code with example usage: obj.additional_update_for_module() and returns: results