(default: "False").
Example:
  >>> import os
  >>> import torch
  >>> import torch.distributed as dist
  >>> import torch.multiprocessing as mp
  >>> import torch.nn.parallel.DistributedDataParallel as DDP
  >>> import torch.distributed.optim.ZeroRedundancyOptimizer as ZeRO
  >>> from torch.distributed.algorithms.join import Join
  >>>
  >>> # On each spawned worker
  >>> def worker(rank):
  >>>     dist.init_process_group("nccl", rank=rank, world_size=2)
  >>>     model = DDP(torch.nn.Linear(1, 1).to(rank), device_ids=[rank])
  >>>     optim = ZeRO(model.parameters(), torch.optim.Adam, lr=0.01)
  >>>     # Rank 1 gets one more input than rank 0
  >>>     inputs = [torch.tensor([1.]).to(rank) for _ in range(10 + rank)]
  >>>     with Join([model, optim]):
  >>>         for input in inputs:
  >>>             loss = model(input).sum()
  >>>             loss.backward()
  >>>             optim.step()
