class_docstr: Implements APPO loss / update logic on top of ImpalaTorchLearner.
class_docstr_tok: ["Implements","APPO","loss","\/","update","logic","on","top","of","ImpalaTorchLearner","."]
docstr: Logic for (maybe) making all Modules within self._module DDP.
  
  This implementation differs from the super's default one in using the special
  TorchDDPRLModuleWithTargetNetworksInterface wrapper, instead of the default
  TorchDDPRLModule one.
docstr_tok: ["Logic","for","(","maybe",")","making","all","Modules","within","self._module","DDP",".","This","implementation","differs","from","the","super","'s","default","one","in","using","the","special","TorchDDPRLModuleWithTargetNetworksInterface","wrapper",",","instead","of","the","default","TorchDDPRLModule","one","."]
code: def _make_modules_ddp_if_necessary(self) -> None:
      """Logic for (maybe) making all Modules within self._module DDP.
  
      This implementation differs from the super's default one in using the special
      TorchDDPRLModuleWithTargetNetworksInterface wrapper, instead of the default
      TorchDDPRLModule one.
      """
  
      # If the module is a MultiAgentRLModule and nn.Module we can simply assume
      # all the submodules are registered. Otherwise, we need to loop through
      # each submodule and move it to the correct device.
      # TODO (Kourosh): This can result in missing modules if the user does not
      #  register them in the MultiAgentRLModule. We should find a better way to
      #  handle this.
      if self._distributed:
          # Single agent module: Convert to
          # `TorchDDPRLModuleWithTargetNetworksInterface`.
          if isinstance(
              self._module, RLModuleWithTargetNetworksInterface
          ):
              self._module = (
                  TorchDDPRLModuleWithTargetNetworksInterface(
                      self._module
                  )
              )
          # Multi agent module: Convert each submodule to
          # `TorchDDPRLModuleWithTargetNetworksInterface`.
          else:
              assert isinstance(self._module, MultiAgentRLModule)
              for key in self._module.keys():
                  sub_module = self._module[key]
                  if isinstance(sub_module, TorchRLModule):
                      # Wrap and override the module ID key in self._module.
                      self._module.add_module(
                          key,
                          TorchDDPRLModuleWithTargetNetworksInterface(
                              sub_module
                          ),
                          override=True,
                      )
  
imports: from typing import Any, Dict
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.algorithms.appo.appo import APPOConfig, LEARNER_RESULTS_CURR_KL_COEFF_KEY, LEARNER_RESULTS_KL_KEY, OLD_ACTION_DIST_LOGITS_KEY
  from ray.rllib.algorithms.appo.appo_learner import AppoLearner
  from ray.rllib.algorithms.impala.torch.vtrace_torch_v2 import make_time_major, vtrace_torch
  from ray.rllib.core.learner.learner import POLICY_LOSS_KEY, VF_LOSS_KEY, ENTROPY_KEY
  from ray.rllib.core.learner.torch.torch_learner import TorchLearner
  from ray.rllib.core.rl_module.marl_module import MultiAgentRLModule
  from ray.rllib.core.rl_module.torch.torch_rl_module import TorchDDPRLModuleWithTargetNetworksInterface, TorchRLModule
  from ray.rllib.core.rl_module.rl_module_with_target_networks_interface import RLModuleWithTargetNetworksInterface
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.nested_dict import NestedDict
  from ray.rllib.utils.typing import ModuleID, TensorType
label_desc: Use image node_id 2 for calling the APPOTorchLearner obj's underlying member method code with example usage: obj._make_modules_ddp_if_necessary() without return types