class_docstr: Strategy for multi-process single-device training on one or multiple nodes.
class_docstr_tok: ["Strategy","for","multi-process","single-device","training","on","one","or","multiple","nodes","."]
docstr: Reduces a tensor from several distributed processes to one aggregated tensor.
  
  Args:
      tensor: the tensor to sync and reduce
      group: the process group to gather results from. Defaults to all processes (world)
      reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
          Can also be a string 'sum' to calculate the sum during reduction.
  
  Return:
      reduced value, except when the input was not a tensor the output remains is unchanged
docstr_tok: ["Reduces","a","tensor","from","several","distributed","processes","to","one","aggregated","tensor",".","Args",":","tensor",":","the","tensor","to","sync","and","reduce","group",":","the","process","group","to","gather","results","from",".","Defaults","to","all","processes","(","world",")","reduce_op",":","the","reduction","operation",".","Defaults","to","'mean'\/'avg","'",".","Can","also","be","a","string","'sum","'","to","calculate","the","sum","during","reduction",".","Return",":","reduced","value",",","except","when","the","input","was","not","a","tensor","the","output","remains","is","unchanged"]
code: def all_reduce(
      self,
      tensor: Tensor,
      group: Optional[Any] = None,
      reduce_op: Optional[Union[ReduceOp, str]] = "mean",
  ) -> Tensor:
      """Reduces a tensor from several distributed processes to one aggregated tensor.
  
      Args:
          tensor: the tensor to sync and reduce
          group: the process group to gather results from. Defaults to all processes (world)
          reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
              Can also be a string 'sum' to calculate the sum during reduction.
  
      Return:
          reduced value, except when the input was not a tensor the output remains is unchanged
  
      """
      if isinstance(tensor, Tensor):
          return _sync_ddp_if_available(
              tensor, group, reduce_op=reduce_op
          )
      return tensor
  
imports: from contextlib import nullcontext
  from datetime import timedelta
  from typing import Any, ContextManager, Dict, List, Literal, Optional, Union
  import torch
  import torch.distributed
  from lightning_utilities.core.rank_zero import rank_zero_only
  from torch import Tensor
  from torch.nn import Module
  from torch.nn.parallel.distributed import DistributedDataParallel
  from typing_extensions import override
  from lightning.fabric.accelerators.accelerator import Accelerator
  from lightning.fabric.plugins.collectives.torch_collective import default_pg_timeout
  from lightning.fabric.plugins.environments.cluster_environment import ClusterEnvironment
  from lightning.fabric.plugins.io.checkpoint_io import CheckpointIO
  from lightning.fabric.plugins.precision import Precision
  from lightning.fabric.strategies.launchers.multiprocessing import _MultiProcessingLauncher
  from lightning.fabric.strategies.launchers.subprocess_script import _SubprocessScriptLauncher
  from lightning.fabric.strategies.parallel import ParallelStrategy
  from lightning.fabric.strategies.registry import _StrategyRegistry
  from lightning.fabric.strategies.strategy import TBroadcast, _BackwardSyncControl
  from lightning.fabric.utilities.distributed import ReduceOp, _distributed_is_initialized, _get_default_process_group_backend_for_device, _init_dist_connection, _sync_ddp_if_available
  from lightning.fabric.utilities.distributed import group
  from lightning.fabric.utilities.rank_zero import rank_zero_only
label_desc: Use image node_id 12 for calling the DDPStrategy obj's underlying member method code with example usage: obj.all_reduce(tensor, group, reduce_op) and returns: tensor, _sync_ddp_if_available