We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Auto dataloader - ValueError: batch_size should be a positive integer value, but got batch_size=0
## 🐛 Bug description

The following code will brake if `batch_size` is smaller than world size :
```python
import torch
import ignite.distributed as idist

def foo(i):
    data = torch.arange(100).reshape(25, 4)
    data_loader = idist.auto_dataloader(data, batch_size=6, num_workers=12)


if __name__ == "__main__":
    idist.spawn("gloo", foo, args=(), nproc_per_node=8)    
```
```
ValueError: batch_size should be a positive integer value, but got batch_size=0
```

A fix can be:
- keep batch size as provided if smaller than world size
- same for `num_workers`

## Environment

 - PyTorch Version (e.g., 1.4): 1.5.0
 - Ignite Version (e.g., 0.3.0): master
 - OS (e.g., Linux): linux
 - How you installed Ignite (`conda`, `pip`, source):
 - Python version: 3.7
 - Any other relevant information:


@InCogNiTo124 would you like to fix this as you recently played around `auto_*` ?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `ignite/distributed/auto.py`
Content:
```
1 import warnings
2 
3 import torch
4 import torch.nn as nn
5 from torch.optim.optimizer import Optimizer
6 from torch.utils.data import DataLoader, Dataset
7 from torch.utils.data.distributed import DistributedSampler
8 from torch.utils.data.sampler import Sampler
9 
10 from ignite.distributed import utils as idist
11 from ignite.distributed.comp_models import native as idist_native
12 from ignite.distributed.comp_models import xla as idist_xla
13 from ignite.utils import setup_logger
14 
15 __all__ = ["auto_dataloader", "auto_model", "auto_optim", "DistributedProxySampler"]
16 
17 
18 def auto_dataloader(dataset, **kwargs):
19     """Helper method to create a dataloader adapted for non-distributed and distributed configurations (supporting
20     all available backends from :meth:`~ignite.distributed.utils.available_backends()`).
21 
22     Internally, we create a dataloader with provided kwargs while applying the following updates:
23 
24     - batch size is scaled by world size: ``batch_size / world_size``.
25     - number of workers is scaled by number of local processes: ``num_workers / nprocs``.
26     - if no sampler provided by user, `torch DistributedSampler` is setup.
27     - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.
28     - if the default device is 'cuda', `pin_memory` is automatically set to `True`.
29 
30     .. warning::
31 
32         Custom batch sampler is not adapted for distributed configuration. Please, make sure that provided batch
33         sampler is compatible with distributed configuration.
34 
35     Examples:
36 
37     .. code-block:: python
38 
39         import ignite.distribted as idist
40 
41         train_loader = idist.auto_dataloader(
42             train_dataset,
43             batch_size=32,
44             num_workers=4,
45             shuffle=True,
46             pin_memory="cuda" in idist.device().type,
47             drop_last=True,
48         )
49 
50     Args:
51         dataset (Dataset): input torch dataset
52         **kwargs: keyword arguments for `torch DataLoader`_.
53 
54     Returns:
55         `torch DataLoader`_ or `XLA MpDeviceLoader`_ for XLA devices
56 
57     .. _torch DataLoader: https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader
58     .. _XLA MpDeviceLoader: https://github.com/pytorch/xla/blob/master/torch_xla/distributed/parallel_loader.py#L178
59     .. _torch DistributedSampler:
60         https://pytorch.org/docs/stable/data.html#torch.utils.data.distributed.DistributedSampler
61     """
62     rank = idist.get_rank()
63     world_size = idist.get_world_size()
64 
65     logger = setup_logger(__name__ + ".auto_dataloader")
66 
67     if world_size > 1:
68         if "batch_size" in kwargs:
69             kwargs["batch_size"] //= world_size
70 
71         if "num_workers" in kwargs:
72             nproc = idist.get_nproc_per_node()
73             kwargs["num_workers"] = (kwargs["num_workers"] + nproc - 1) // nproc
74 
75         if "batch_sampler" not in kwargs:
76             if kwargs.get("sampler", None) is not None:
77                 sampler = DistributedProxySampler(kwargs["sampler"], num_replicas=world_size, rank=rank)
78             else:
79                 sampler = DistributedSampler(
80                     dataset, num_replicas=world_size, rank=rank, shuffle=kwargs.get("shuffle", True)
81                 )
82                 # we need to remove "shuffle" from kwargs if sampler is used
83                 if "shuffle" in kwargs:
84                     del kwargs["shuffle"]
85 
86             kwargs["sampler"] = sampler
87         else:
88             warnings.warn(
89                 "Found batch_sampler in provided kwargs. Please, make sure that it is compatible "
90                 "with distributed configuration"
91             )
92 
93     if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and kwargs.get("pin_memory", False):
94         # TODO: How about XLA GPU ?
95         warnings.warn(
96             "Found incompatible options: xla support and pin_memory args equal True. "
97             "Argument `pin_memory=False` will be used to construct data loader."
98         )
99         kwargs["pin_memory"] = False
100     else:
101         kwargs["pin_memory"] = kwargs.get("pin_memory", "cuda" in idist.device().type)
102 
103     logger.info("Use data loader kwargs for dataset '{}': \n\t{}".format(repr(dataset)[:20].strip(), kwargs))
104     dataloader = DataLoader(dataset, **kwargs)
105 
106     if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and world_size > 1:
107 
108         logger.info("DataLoader is wrapped by `MpDeviceLoader` on XLA")
109 
110         mp_device_loader_cls = _MpDeviceLoader
111         try:
112             from torch_xla.distributed.parallel_loader import MpDeviceLoader
113 
114             mp_device_loader_cls = MpDeviceLoader
115         except ImportError:
116             pass
117 
118         sampler = dataloader.sampler
119         dataloader = mp_device_loader_cls(dataloader, idist.device())
120         dataloader.sampler = sampler
121 
122     return dataloader
123 
124 
125 def auto_model(model: nn.Module) -> nn.Module:
126     """Helper method to adapt provided model for non-distributed and distributed configurations (supporting
127     all available backends from :meth:`~ignite.distributed.utils.available_backends()`).
128 
129     Internally, we perform to following:
130 
131     - send model to current :meth:`~ignite.distributed.utils.device()`.
132     - wrap the model to `torch DistributedDataParallel`_ for native torch distributed if world size is larger than 1
133     - wrap the model to `torch DataParallel`_ if no distributed context found and more than one CUDA devices available.
134 
135     Examples:
136 
137     .. code-block:: python
138 
139         import ignite.distribted as idist
140 
141         model = idist.auto_model(model)
142 
143     Args:
144         model (torch.nn.Module): model to adapt.
145 
146     Returns:
147         torch.nn.Module
148 
149     .. _torch DistributedDataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.parallel.DistributedDataParallel
150     .. _torch DataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.DataParallel
151     """
152     logger = setup_logger(__name__ + ".auto_model")
153 
154     model.to(idist.device())
155 
156     # distributed data parallel model
157     if idist.get_world_size() > 1:
158         if idist.backend() == idist_native.NCCL:
159             lrank = idist.get_local_rank()
160             logger.info("Apply torch DistributedDataParallel on model, device id: {}".format(lrank))
161             model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[lrank,])
162         elif idist.backend() == idist_native.GLOO:
163             logger.info("Apply torch DistributedDataParallel on model")
164             model = torch.nn.parallel.DistributedDataParallel(model)
165 
166     # not distributed but multiple GPUs reachable so data parallel model
167     elif torch.cuda.device_count() > 1 and "cuda" in idist.device().type:
168         logger.info("Apply torch DataParallel on model")
169         model = torch.nn.parallel.DataParallel(model)
170 
171     return model
172 
173 
174 def auto_optim(optimizer: Optimizer) -> Optimizer:
175     """Helper method to adapt optimizer for non-distributed and distributed configurations (supporting
176     all available backends from :meth:`~ignite.distributed.utils.available_backends()`).
177 
178     Internally, this method is no-op for non-distributed and torch native distributed configuration.
179     For XLA distributed configuration, we create a new class that inherits from provided optimizer.
180     The goal is to override the `step()` method with specific `xm.optimizer_step`_ implementation.
181 
182     Examples:
183 
184     .. code-block:: python
185 
186         import ignite.distribted as idist
187 
188         optimizer = idist.auto_optim(optimizer)
189 
190 
191     Args:
192         optimizer (Optimizer): input torch optimizer
193 
194     Returns:
195         Optimizer
196 
197     .. _xm.optimizer_step: http://pytorch.org/xla/release/1.5/index.html#torch_xla.core.xla_model.optimizer_step
198 
199     """
200     if not (idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU):
201         return optimizer
202 
203     cls = type(optimizer.__class__.__name__, (optimizer.__class__,), dict(_XLADistributedOptimizer.__dict__))
204     return cls(optimizer)
205 
206 
207 class DistributedProxySampler(DistributedSampler):
208     """Distributed sampler proxy to adapt user's sampler for distributed data parallelism configuration.
209 
210     Code is based on https://github.com/pytorch/pytorch/issues/23430#issuecomment-562350407
211 
212 
213     .. note::
214         Input sampler is assumed to have a constant size.
215 
216     Args:
217         sampler (Sampler): Input torch data sampler.
218         num_replicas (int, optional): Number of processes participating in distributed training.
219         rank (int, optional): Rank of the current process within ``num_replicas``.
220 
221     """
222 
223     def __init__(self, sampler: Sampler, num_replicas=None, rank=None):
224 
225         if not isinstance(sampler, Sampler):
226             raise TypeError("Argument sampler should be instance of torch Sampler, but given: {}".format(type(sampler)))
227 
228         if not hasattr(sampler, "__len__"):
229             raise TypeError("Argument sampler should have length")
230 
231         super(DistributedProxySampler, self).__init__(sampler, num_replicas=num_replicas, rank=rank, shuffle=False)
232         self.sampler = sampler
233 
234     def __iter__(self):
235         # deterministically shuffle based on epoch
236         torch.manual_seed(self.epoch)
237 
238         indices = []
239         while len(indices) < self.total_size:
240             indices += list(self.sampler)
241 
242         if len(indices) != self.total_size:
243             raise RuntimeError("{} vs {}".format(len(indices), self.total_size))
244 
245         # subsample
246         indices = indices[self.rank : self.total_size : self.num_replicas]
247         if len(indices) != self.num_samples:
248             raise RuntimeError("{} vs {}".format(len(indices), self.num_samples))
249 
250         return iter(indices)
251 
252 
253 if idist.has_xla_support:
254 
255     import torch_xla.core.xla_model as xm
256     from torch_xla.distributed.parallel_loader import ParallelLoader
257 
258     class _MpDeviceLoader:
259         # https://github.com/pytorch/xla/pull/2117
260         # From pytorch/xla if `torch_xla.distributed.parallel_loader.MpDeviceLoader` is not available
261         def __init__(self, loader, device, **kwargs):
262             self._loader = loader
263             self._device = device
264             self._parallel_loader_kwargs = kwargs
265 
266         def __iter__(self):
267             parallel_loader = ParallelLoader(self._loader, [self._device], **self._parallel_loader_kwargs)
268             return parallel_loader.per_device_loader(self._device)
269 
270         def __len__(self):
271             return len(self._loader)
272 
273     class _XLADistributedOptimizer(Optimizer):
274         def __init__(self, optimizer):
275             super(self.__class__, self).__init__(optimizer.param_groups)
276             self.wrapped_optimizer = optimizer
277 
278         def step(self, closure=None):
279             xm.optimizer_step(self.wrapped_optimizer, barrier=True)
280
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/ignite/distributed/auto.py b/ignite/distributed/auto.py
--- a/ignite/distributed/auto.py
+++ b/ignite/distributed/auto.py
@@ -21,8 +21,8 @@
 
     Internally, we create a dataloader with provided kwargs while applying the following updates:
 
-    - batch size is scaled by world size: ``batch_size / world_size``.
-    - number of workers is scaled by number of local processes: ``num_workers / nprocs``.
+    - batch size is scaled by world size: ``batch_size / world_size`` if larger or equal world size.
+    - number of workers is scaled by number of local processes: ``num_workers / nprocs`` if larger or equal world size.
     - if no sampler provided by user, `torch DistributedSampler` is setup.
     - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.
     - if the default device is 'cuda', `pin_memory` is automatically set to `True`.
@@ -63,13 +63,12 @@
     world_size = idist.get_world_size()
 
     logger = setup_logger(__name__ + ".auto_dataloader")
-
     if world_size > 1:
-        if "batch_size" in kwargs:
+        if "batch_size" in kwargs and kwargs["batch_size"] >= world_size:
             kwargs["batch_size"] //= world_size
 
-        if "num_workers" in kwargs:
-            nproc = idist.get_nproc_per_node()
+        nproc = idist.get_nproc_per_node()
+        if "num_workers" in kwargs and kwargs["num_workers"] >= nproc:
             kwargs["num_workers"] = (kwargs["num_workers"] + nproc - 1) // nproc
 
         if "batch_sampler" not in kwargs:

{"golden_diff": "diff --git a/ignite/distributed/auto.py b/ignite/distributed/auto.py\n--- a/ignite/distributed/auto.py\n+++ b/ignite/distributed/auto.py\n@@ -21,8 +21,8 @@\n \n     Internally, we create a dataloader with provided kwargs while applying the following updates:\n \n-    - batch size is scaled by world size: ``batch_size / world_size``.\n-    - number of workers is scaled by number of local processes: ``num_workers / nprocs``.\n+    - batch size is scaled by world size: ``batch_size / world_size`` if larger or equal world size.\n+    - number of workers is scaled by number of local processes: ``num_workers / nprocs`` if larger or equal world size.\n     - if no sampler provided by user, `torch DistributedSampler` is setup.\n     - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.\n     - if the default device is 'cuda', `pin_memory` is automatically set to `True`.\n@@ -63,13 +63,12 @@\n     world_size = idist.get_world_size()\n \n     logger = setup_logger(__name__ + \".auto_dataloader\")\n-\n     if world_size > 1:\n-        if \"batch_size\" in kwargs:\n+        if \"batch_size\" in kwargs and kwargs[\"batch_size\"] >= world_size:\n             kwargs[\"batch_size\"] //= world_size\n \n-        if \"num_workers\" in kwargs:\n-            nproc = idist.get_nproc_per_node()\n+        nproc = idist.get_nproc_per_node()\n+        if \"num_workers\" in kwargs and kwargs[\"num_workers\"] >= nproc:\n             kwargs[\"num_workers\"] = (kwargs[\"num_workers\"] + nproc - 1) // nproc\n \n         if \"batch_sampler\" not in kwargs:\n", "issue": "Auto dataloader - ValueError: batch_size should be a positive integer value, but got batch_size=0\n## \ud83d\udc1b Bug description\r\n\r\nThe following code will brake if `batch_size` is smaller than world size :\r\n```python\r\nimport torch\r\nimport ignite.distributed as idist\r\n\r\ndef foo(i):\r\n    data = torch.arange(100).reshape(25, 4)\r\n    data_loader = idist.auto_dataloader(data, batch_size=6, num_workers=12)\r\n\r\n\r\nif __name__ == \"__main__\":\r\n    idist.spawn(\"gloo\", foo, args=(), nproc_per_node=8)    \r\n```\r\n```\r\nValueError: batch_size should be a positive integer value, but got batch_size=0\r\n```\r\n\r\nA fix can be:\r\n- keep batch size as provided if smaller than world size\r\n- same for `num_workers`\r\n\r\n## Environment\r\n\r\n - PyTorch Version (e.g., 1.4): 1.5.0\r\n - Ignite Version (e.g., 0.3.0): master\r\n - OS (e.g., Linux): linux\r\n - How you installed Ignite (`conda`, `pip`, source):\r\n - Python version: 3.7\r\n - Any other relevant information:\r\n\r\n\r\n@InCogNiTo124 would you like to fix this as you recently played around `auto_*` ?\n", "before_files": [{"content": "import warnings\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim.optimizer import Optimizer\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.data.sampler import Sampler\n\nfrom ignite.distributed import utils as idist\nfrom ignite.distributed.comp_models import native as idist_native\nfrom ignite.distributed.comp_models import xla as idist_xla\nfrom ignite.utils import setup_logger\n\n__all__ = [\"auto_dataloader\", \"auto_model\", \"auto_optim\", \"DistributedProxySampler\"]\n\n\ndef auto_dataloader(dataset, **kwargs):\n    \"\"\"Helper method to create a dataloader adapted for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, we create a dataloader with provided kwargs while applying the following updates:\n\n    - batch size is scaled by world size: ``batch_size / world_size``.\n    - number of workers is scaled by number of local processes: ``num_workers / nprocs``.\n    - if no sampler provided by user, `torch DistributedSampler` is setup.\n    - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.\n    - if the default device is 'cuda', `pin_memory` is automatically set to `True`.\n\n    .. warning::\n\n        Custom batch sampler is not adapted for distributed configuration. Please, make sure that provided batch\n        sampler is compatible with distributed configuration.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        train_loader = idist.auto_dataloader(\n            train_dataset,\n            batch_size=32,\n            num_workers=4,\n            shuffle=True,\n            pin_memory=\"cuda\" in idist.device().type,\n            drop_last=True,\n        )\n\n    Args:\n        dataset (Dataset): input torch dataset\n        **kwargs: keyword arguments for `torch DataLoader`_.\n\n    Returns:\n        `torch DataLoader`_ or `XLA MpDeviceLoader`_ for XLA devices\n\n    .. _torch DataLoader: https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader\n    .. _XLA MpDeviceLoader: https://github.com/pytorch/xla/blob/master/torch_xla/distributed/parallel_loader.py#L178\n    .. _torch DistributedSampler:\n        https://pytorch.org/docs/stable/data.html#torch.utils.data.distributed.DistributedSampler\n    \"\"\"\n    rank = idist.get_rank()\n    world_size = idist.get_world_size()\n\n    logger = setup_logger(__name__ + \".auto_dataloader\")\n\n    if world_size > 1:\n        if \"batch_size\" in kwargs:\n            kwargs[\"batch_size\"] //= world_size\n\n        if \"num_workers\" in kwargs:\n            nproc = idist.get_nproc_per_node()\n            kwargs[\"num_workers\"] = (kwargs[\"num_workers\"] + nproc - 1) // nproc\n\n        if \"batch_sampler\" not in kwargs:\n            if kwargs.get(\"sampler\", None) is not None:\n                sampler = DistributedProxySampler(kwargs[\"sampler\"], num_replicas=world_size, rank=rank)\n            else:\n                sampler = DistributedSampler(\n                    dataset, num_replicas=world_size, rank=rank, shuffle=kwargs.get(\"shuffle\", True)\n                )\n                # we need to remove \"shuffle\" from kwargs if sampler is used\n                if \"shuffle\" in kwargs:\n                    del kwargs[\"shuffle\"]\n\n            kwargs[\"sampler\"] = sampler\n        else:\n            warnings.warn(\n                \"Found batch_sampler in provided kwargs. Please, make sure that it is compatible \"\n                \"with distributed configuration\"\n            )\n\n    if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and kwargs.get(\"pin_memory\", False):\n        # TODO: How about XLA GPU ?\n        warnings.warn(\n            \"Found incompatible options: xla support and pin_memory args equal True. \"\n            \"Argument `pin_memory=False` will be used to construct data loader.\"\n        )\n        kwargs[\"pin_memory\"] = False\n    else:\n        kwargs[\"pin_memory\"] = kwargs.get(\"pin_memory\", \"cuda\" in idist.device().type)\n\n    logger.info(\"Use data loader kwargs for dataset '{}': \\n\\t{}\".format(repr(dataset)[:20].strip(), kwargs))\n    dataloader = DataLoader(dataset, **kwargs)\n\n    if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and world_size > 1:\n\n        logger.info(\"DataLoader is wrapped by `MpDeviceLoader` on XLA\")\n\n        mp_device_loader_cls = _MpDeviceLoader\n        try:\n            from torch_xla.distributed.parallel_loader import MpDeviceLoader\n\n            mp_device_loader_cls = MpDeviceLoader\n        except ImportError:\n            pass\n\n        sampler = dataloader.sampler\n        dataloader = mp_device_loader_cls(dataloader, idist.device())\n        dataloader.sampler = sampler\n\n    return dataloader\n\n\ndef auto_model(model: nn.Module) -> nn.Module:\n    \"\"\"Helper method to adapt provided model for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, we perform to following:\n\n    - send model to current :meth:`~ignite.distributed.utils.device()`.\n    - wrap the model to `torch DistributedDataParallel`_ for native torch distributed if world size is larger than 1\n    - wrap the model to `torch DataParallel`_ if no distributed context found and more than one CUDA devices available.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        model = idist.auto_model(model)\n\n    Args:\n        model (torch.nn.Module): model to adapt.\n\n    Returns:\n        torch.nn.Module\n\n    .. _torch DistributedDataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.parallel.DistributedDataParallel\n    .. _torch DataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.DataParallel\n    \"\"\"\n    logger = setup_logger(__name__ + \".auto_model\")\n\n    model.to(idist.device())\n\n    # distributed data parallel model\n    if idist.get_world_size() > 1:\n        if idist.backend() == idist_native.NCCL:\n            lrank = idist.get_local_rank()\n            logger.info(\"Apply torch DistributedDataParallel on model, device id: {}\".format(lrank))\n            model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[lrank,])\n        elif idist.backend() == idist_native.GLOO:\n            logger.info(\"Apply torch DistributedDataParallel on model\")\n            model = torch.nn.parallel.DistributedDataParallel(model)\n\n    # not distributed but multiple GPUs reachable so data parallel model\n    elif torch.cuda.device_count() > 1 and \"cuda\" in idist.device().type:\n        logger.info(\"Apply torch DataParallel on model\")\n        model = torch.nn.parallel.DataParallel(model)\n\n    return model\n\n\ndef auto_optim(optimizer: Optimizer) -> Optimizer:\n    \"\"\"Helper method to adapt optimizer for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, this method is no-op for non-distributed and torch native distributed configuration.\n    For XLA distributed configuration, we create a new class that inherits from provided optimizer.\n    The goal is to override the `step()` method with specific `xm.optimizer_step`_ implementation.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        optimizer = idist.auto_optim(optimizer)\n\n\n    Args:\n        optimizer (Optimizer): input torch optimizer\n\n    Returns:\n        Optimizer\n\n    .. _xm.optimizer_step: http://pytorch.org/xla/release/1.5/index.html#torch_xla.core.xla_model.optimizer_step\n\n    \"\"\"\n    if not (idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU):\n        return optimizer\n\n    cls = type(optimizer.__class__.__name__, (optimizer.__class__,), dict(_XLADistributedOptimizer.__dict__))\n    return cls(optimizer)\n\n\nclass DistributedProxySampler(DistributedSampler):\n    \"\"\"Distributed sampler proxy to adapt user's sampler for distributed data parallelism configuration.\n\n    Code is based on https://github.com/pytorch/pytorch/issues/23430#issuecomment-562350407\n\n\n    .. note::\n        Input sampler is assumed to have a constant size.\n\n    Args:\n        sampler (Sampler): Input torch data sampler.\n        num_replicas (int, optional): Number of processes participating in distributed training.\n        rank (int, optional): Rank of the current process within ``num_replicas``.\n\n    \"\"\"\n\n    def __init__(self, sampler: Sampler, num_replicas=None, rank=None):\n\n        if not isinstance(sampler, Sampler):\n            raise TypeError(\"Argument sampler should be instance of torch Sampler, but given: {}\".format(type(sampler)))\n\n        if not hasattr(sampler, \"__len__\"):\n            raise TypeError(\"Argument sampler should have length\")\n\n        super(DistributedProxySampler, self).__init__(sampler, num_replicas=num_replicas, rank=rank, shuffle=False)\n        self.sampler = sampler\n\n    def __iter__(self):\n        # deterministically shuffle based on epoch\n        torch.manual_seed(self.epoch)\n\n        indices = []\n        while len(indices) < self.total_size:\n            indices += list(self.sampler)\n\n        if len(indices) != self.total_size:\n            raise RuntimeError(\"{} vs {}\".format(len(indices), self.total_size))\n\n        # subsample\n        indices = indices[self.rank : self.total_size : self.num_replicas]\n        if len(indices) != self.num_samples:\n            raise RuntimeError(\"{} vs {}\".format(len(indices), self.num_samples))\n\n        return iter(indices)\n\n\nif idist.has_xla_support:\n\n    import torch_xla.core.xla_model as xm\n    from torch_xla.distributed.parallel_loader import ParallelLoader\n\n    class _MpDeviceLoader:\n        # https://github.com/pytorch/xla/pull/2117\n        # From pytorch/xla if `torch_xla.distributed.parallel_loader.MpDeviceLoader` is not available\n        def __init__(self, loader, device, **kwargs):\n            self._loader = loader\n            self._device = device\n            self._parallel_loader_kwargs = kwargs\n\n        def __iter__(self):\n            parallel_loader = ParallelLoader(self._loader, [self._device], **self._parallel_loader_kwargs)\n            return parallel_loader.per_device_loader(self._device)\n\n        def __len__(self):\n            return len(self._loader)\n\n    class _XLADistributedOptimizer(Optimizer):\n        def __init__(self, optimizer):\n            super(self.__class__, self).__init__(optimizer.param_groups)\n            self.wrapped_optimizer = optimizer\n\n        def step(self, closure=None):\n            xm.optimizer_step(self.wrapped_optimizer, barrier=True)\n", "path": "ignite/distributed/auto.py"}], "after_files": [{"content": "import warnings\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim.optimizer import Optimizer\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.data.sampler import Sampler\n\nfrom ignite.distributed import utils as idist\nfrom ignite.distributed.comp_models import native as idist_native\nfrom ignite.distributed.comp_models import xla as idist_xla\nfrom ignite.utils import setup_logger\n\n__all__ = [\"auto_dataloader\", \"auto_model\", \"auto_optim\", \"DistributedProxySampler\"]\n\n\ndef auto_dataloader(dataset, **kwargs):\n    \"\"\"Helper method to create a dataloader adapted for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, we create a dataloader with provided kwargs while applying the following updates:\n\n    - batch size is scaled by world size: ``batch_size / world_size`` if larger or equal world size.\n    - number of workers is scaled by number of local processes: ``num_workers / nprocs`` if larger or equal world size.\n    - if no sampler provided by user, `torch DistributedSampler` is setup.\n    - if a sampler is provided by user, it is wrapped by :class:`~ignite.distributed.auto.DistributedProxySampler`.\n    - if the default device is 'cuda', `pin_memory` is automatically set to `True`.\n\n    .. warning::\n\n        Custom batch sampler is not adapted for distributed configuration. Please, make sure that provided batch\n        sampler is compatible with distributed configuration.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        train_loader = idist.auto_dataloader(\n            train_dataset,\n            batch_size=32,\n            num_workers=4,\n            shuffle=True,\n            pin_memory=\"cuda\" in idist.device().type,\n            drop_last=True,\n        )\n\n    Args:\n        dataset (Dataset): input torch dataset\n        **kwargs: keyword arguments for `torch DataLoader`_.\n\n    Returns:\n        `torch DataLoader`_ or `XLA MpDeviceLoader`_ for XLA devices\n\n    .. _torch DataLoader: https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader\n    .. _XLA MpDeviceLoader: https://github.com/pytorch/xla/blob/master/torch_xla/distributed/parallel_loader.py#L178\n    .. _torch DistributedSampler:\n        https://pytorch.org/docs/stable/data.html#torch.utils.data.distributed.DistributedSampler\n    \"\"\"\n    rank = idist.get_rank()\n    world_size = idist.get_world_size()\n\n    logger = setup_logger(__name__ + \".auto_dataloader\")\n    if world_size > 1:\n        if \"batch_size\" in kwargs and kwargs[\"batch_size\"] >= world_size:\n            kwargs[\"batch_size\"] //= world_size\n\n        nproc = idist.get_nproc_per_node()\n        if \"num_workers\" in kwargs and kwargs[\"num_workers\"] >= nproc:\n            kwargs[\"num_workers\"] = (kwargs[\"num_workers\"] + nproc - 1) // nproc\n\n        if \"batch_sampler\" not in kwargs:\n            if kwargs.get(\"sampler\", None) is not None:\n                sampler = DistributedProxySampler(kwargs[\"sampler\"], num_replicas=world_size, rank=rank)\n            else:\n                sampler = DistributedSampler(\n                    dataset, num_replicas=world_size, rank=rank, shuffle=kwargs.get(\"shuffle\", True)\n                )\n                # we need to remove \"shuffle\" from kwargs if sampler is used\n                if \"shuffle\" in kwargs:\n                    del kwargs[\"shuffle\"]\n\n            kwargs[\"sampler\"] = sampler\n        else:\n            warnings.warn(\n                \"Found batch_sampler in provided kwargs. Please, make sure that it is compatible \"\n                \"with distributed configuration\"\n            )\n\n    if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and kwargs.get(\"pin_memory\", False):\n        # TODO: How about XLA GPU ?\n        warnings.warn(\n            \"Found incompatible options: xla support and pin_memory args equal True. \"\n            \"Argument `pin_memory=False` will be used to construct data loader.\"\n        )\n        kwargs[\"pin_memory\"] = False\n    else:\n        kwargs[\"pin_memory\"] = kwargs.get(\"pin_memory\", \"cuda\" in idist.device().type)\n\n    logger.info(\"Use data loader kwargs for dataset '{}': \\n\\t{}\".format(repr(dataset)[:20].strip(), kwargs))\n    dataloader = DataLoader(dataset, **kwargs)\n\n    if idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU and world_size > 1:\n\n        logger.info(\"DataLoader is wrapped by `MpDeviceLoader` on XLA\")\n\n        mp_device_loader_cls = _MpDeviceLoader\n        try:\n            from torch_xla.distributed.parallel_loader import MpDeviceLoader\n\n            mp_device_loader_cls = MpDeviceLoader\n        except ImportError:\n            pass\n\n        sampler = dataloader.sampler\n        dataloader = mp_device_loader_cls(dataloader, idist.device())\n        dataloader.sampler = sampler\n\n    return dataloader\n\n\ndef auto_model(model: nn.Module) -> nn.Module:\n    \"\"\"Helper method to adapt provided model for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, we perform to following:\n\n    - send model to current :meth:`~ignite.distributed.utils.device()`.\n    - wrap the model to `torch DistributedDataParallel`_ for native torch distributed if world size is larger than 1\n    - wrap the model to `torch DataParallel`_ if no distributed context found and more than one CUDA devices available.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        model = idist.auto_model(model)\n\n    Args:\n        model (torch.nn.Module): model to adapt.\n\n    Returns:\n        torch.nn.Module\n\n    .. _torch DistributedDataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.parallel.DistributedDataParallel\n    .. _torch DataParallel: https://pytorch.org/docs/stable/nn.html#torch.nn.DataParallel\n    \"\"\"\n    logger = setup_logger(__name__ + \".auto_model\")\n\n    model.to(idist.device())\n\n    # distributed data parallel model\n    if idist.get_world_size() > 1:\n        if idist.backend() == idist_native.NCCL:\n            lrank = idist.get_local_rank()\n            logger.info(\"Apply torch DistributedDataParallel on model, device id: {}\".format(lrank))\n            model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[lrank,])\n        elif idist.backend() == idist_native.GLOO:\n            logger.info(\"Apply torch DistributedDataParallel on model\")\n            model = torch.nn.parallel.DistributedDataParallel(model)\n\n    # not distributed but multiple GPUs reachable so data parallel model\n    elif torch.cuda.device_count() > 1 and \"cuda\" in idist.device().type:\n        logger.info(\"Apply torch DataParallel on model\")\n        model = torch.nn.parallel.DataParallel(model)\n\n    return model\n\n\ndef auto_optim(optimizer: Optimizer) -> Optimizer:\n    \"\"\"Helper method to adapt optimizer for non-distributed and distributed configurations (supporting\n    all available backends from :meth:`~ignite.distributed.utils.available_backends()`).\n\n    Internally, this method is no-op for non-distributed and torch native distributed configuration.\n    For XLA distributed configuration, we create a new class that inherits from provided optimizer.\n    The goal is to override the `step()` method with specific `xm.optimizer_step`_ implementation.\n\n    Examples:\n\n    .. code-block:: python\n\n        import ignite.distribted as idist\n\n        optimizer = idist.auto_optim(optimizer)\n\n\n    Args:\n        optimizer (Optimizer): input torch optimizer\n\n    Returns:\n        Optimizer\n\n    .. _xm.optimizer_step: http://pytorch.org/xla/release/1.5/index.html#torch_xla.core.xla_model.optimizer_step\n\n    \"\"\"\n    if not (idist.has_xla_support and idist.backend() == idist_xla.XLA_TPU):\n        return optimizer\n\n    cls = type(optimizer.__class__.__name__, (optimizer.__class__,), dict(_XLADistributedOptimizer.__dict__))\n    return cls(optimizer)\n\n\nclass DistributedProxySampler(DistributedSampler):\n    \"\"\"Distributed sampler proxy to adapt user's sampler for distributed data parallelism configuration.\n\n    Code is based on https://github.com/pytorch/pytorch/issues/23430#issuecomment-562350407\n\n\n    .. note::\n        Input sampler is assumed to have a constant size.\n\n    Args:\n        sampler (Sampler): Input torch data sampler.\n        num_replicas (int, optional): Number of processes participating in distributed training.\n        rank (int, optional): Rank of the current process within ``num_replicas``.\n\n    \"\"\"\n\n    def __init__(self, sampler: Sampler, num_replicas=None, rank=None):\n\n        if not isinstance(sampler, Sampler):\n            raise TypeError(\"Argument sampler should be instance of torch Sampler, but given: {}\".format(type(sampler)))\n\n        if not hasattr(sampler, \"__len__\"):\n            raise TypeError(\"Argument sampler should have length\")\n\n        super(DistributedProxySampler, self).__init__(sampler, num_replicas=num_replicas, rank=rank, shuffle=False)\n        self.sampler = sampler\n\n    def __iter__(self):\n        # deterministically shuffle based on epoch\n        torch.manual_seed(self.epoch)\n\n        indices = []\n        while len(indices) < self.total_size:\n            indices += list(self.sampler)\n\n        if len(indices) != self.total_size:\n            raise RuntimeError(\"{} vs {}\".format(len(indices), self.total_size))\n\n        # subsample\n        indices = indices[self.rank : self.total_size : self.num_replicas]\n        if len(indices) != self.num_samples:\n            raise RuntimeError(\"{} vs {}\".format(len(indices), self.num_samples))\n\n        return iter(indices)\n\n\nif idist.has_xla_support:\n\n    import torch_xla.core.xla_model as xm\n    from torch_xla.distributed.parallel_loader import ParallelLoader\n\n    class _MpDeviceLoader:\n        # https://github.com/pytorch/xla/pull/2117\n        # From pytorch/xla if `torch_xla.distributed.parallel_loader.MpDeviceLoader` is not available\n        def __init__(self, loader, device, **kwargs):\n            self._loader = loader\n            self._device = device\n            self._parallel_loader_kwargs = kwargs\n\n        def __iter__(self):\n            parallel_loader = ParallelLoader(self._loader, [self._device], **self._parallel_loader_kwargs)\n            return parallel_loader.per_device_loader(self._device)\n\n        def __len__(self):\n            return len(self._loader)\n\n    class _XLADistributedOptimizer(Optimizer):\n        def __init__(self, optimizer):\n            super(self.__class__, self).__init__(optimizer.param_groups)\n            self.wrapped_optimizer = optimizer\n\n        def step(self, closure=None):\n            xm.optimizer_step(self.wrapped_optimizer, barrier=True)\n", "path": "ignite/distributed/auto.py"}]}