code: def scale_learning_rate_for_ddp(
      learning_rate: Union[
          numbers.Rational, Dict[str, numbers.Rational]
      ]
  ) -> Union[float, Dict[str, float]]:
      """
      Scale learning rate with respect to world size for Distributed Data Parallel training.
      Efficient learning rate is WORLD_SIZE * learning rate from config
      For non-distributed runs it is No-Op and returns learning_rate argument as is.
      """
      if not is_dist_avail_and_initialized():
          return learning_rate
  
      scale = float(get_world_size())
      if isinstance(learning_rate, Mapping):
          return dict(
              (k, float(v * scale)) for k, v in learning_rate.items()
          )
      elif isinstance(learning_rate, (float, numbers.Rational)):
          return scale * learning_rate
      raise ValueError(
          f"Got unsupported type {type(learning_rate)} for learning rate. Must be either a mapping or a single scalar."
      )
  
file_path: pytorch-toolbelt/pytorch_toolbelt/optimization/functional.py
imports: import dataclasses
  import numbers
  from typing import Optional, Iterator, Dict, Union, List, Tuple, Mapping
  from torch import nn
  from pytorch_toolbelt.utils.distributed import get_rank, get_world_size, is_dist_avail_and_initialized
  from pytorch_toolbelt.utils.torch_utils import get_non_wrapped_model
label_desc: Use image node_id 1 for calling a global function with example usage: scale_learning_rate_for_ddp(learning_rate) and returns: learning_rate, dict, unknown