We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
multi-gpu ddp calls validation and testing loops too many times
When using ddp with multiple gpus, each validation and test loop is called with the entire validation dataset for each gpu.

Expected behavior is that the dataset is divided appropriately across the gpus.

I am using current master (cloned Mar 14), Ubuntu 19.10, Cuda 10.1, python 3.7.5, pytorch 1.4, venv environment.

The problem appears to be in `auto_add_sampler()` in data_loading.py. It does not create a `DistributedSampler` for validation or test datasets.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/trainer/data_loading.py`
Content:
```
1 from abc import ABC, abstractmethod
2 from typing import Union, List, Tuple, Callable
3 
4 import torch.distributed as torch_distrib
5 from torch.utils.data import SequentialSampler, DataLoader
6 from torch.utils.data.distributed import DistributedSampler
7 
8 from pytorch_lightning.core import LightningModule
9 from pytorch_lightning.utilities.debugging import MisconfigurationException
10 
11 try:
12     from apex import amp
13 except ImportError:
14     APEX_AVAILABLE = False
15 else:
16     APEX_AVAILABLE = True
17 
18 try:
19     import torch_xla
20     import torch_xla.core.xla_model as xm
21     import torch_xla.distributed.xla_multiprocessing as xmp
22 except ImportError:
23     XLA_AVAILABLE = False
24 else:
25     XLA_AVAILABLE = True
26 
27 
28 def _has_len(dataloader: DataLoader) -> bool:
29     try:
30         # try getting the length
31         _ = len(dataloader)
32         return True
33     except TypeError:
34         return False
35 
36 
37 class TrainerDataLoadingMixin(ABC):
38 
39     # this is just a summary on variables used in this abstract class,
40     #  the proper values/initialisation should be done in child class
41     proc_rank: int
42     use_ddp: bool
43     use_ddp2: bool
44     shown_warnings: ...
45     val_check_interval: float
46     use_tpu: bool
47     tpu_local_core_rank: int
48     train_dataloader: DataLoader
49     num_training_batches: Union[int, float]
50     val_check_batch: ...
51     val_dataloaders: List[DataLoader]
52     num_val_batches: Union[int, float]
53     test_dataloaders: List[DataLoader]
54     num_test_batches: Union[int, float]
55     train_percent_check: float
56     val_percent_check: float
57     test_percent_check: float
58 
59     @abstractmethod
60     def is_overriden(self, *args):
61         """Warning: this is just empty shell for code implemented in other class."""
62 
63     def _percent_range_check(self, name: str) -> None:
64         value = getattr(self, name)
65         msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'
66         if name == 'val_check_interval':
67             msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'
68 
69         if not 0. <= value <= 1.:
70             raise ValueError(msg)
71 
72     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
73         if self.use_ddp or self.use_ddp2 or self.use_tpu:
74             dl_args = {
75                 'dataset': dataloader.dataset,
76                 'batch_size': dataloader.batch_size,
77                 'shuffle': False,
78                 'num_workers': dataloader.num_workers,
79                 'collate_fn': dataloader.collate_fn,
80                 'pin_memory': dataloader.pin_memory,
81                 'drop_last': dataloader.drop_last,
82                 'timeout': dataloader.timeout,
83                 'worker_init_fn': dataloader.worker_init_fn
84             }
85 
86             if self.use_tpu:
87                 sampler = DistributedSampler(
88                     dataloader.dataset,
89                     num_replicas=xm.xrt_world_size(),
90                     rank=xm.get_ordinal()
91                 )
92                 dl_args['shuffle'] = False
93             else:
94                 if train:
95                     sampler = DistributedSampler(dataloader.dataset)
96                     dl_args['shuffle'] = False
97                 else:
98                     sampler = SequentialSampler(dataloader.dataset)
99 
100             dl_args['sampler'] = sampler
101 
102             dataloader = DataLoader(**dl_args)
103         return dataloader
104 
105     def reset_train_dataloader(self, model: LightningModule) -> None:
106         """Resets the train dataloader and initialises required variables
107         (number of batches, when to validate, etc.).
108 
109         Args:
110             model: The current `LightningModule`
111         """
112         self.train_dataloader = self.request_dataloader(model.train_dataloader)
113         self.num_training_batches = 0
114 
115         # automatically add samplers
116         self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)
117 
118         self._percent_range_check('train_percent_check')
119 
120         if not _has_len(self.train_dataloader):
121             self.num_training_batches = float('inf')
122         else:
123             # try getting the length
124             self.num_training_batches = len(self.train_dataloader)
125             self.num_training_batches = int(self.num_training_batches * self.train_percent_check)
126 
127         # determine when to check validation
128         # if int passed in, val checks that often
129         # otherwise, it checks in [0, 1.0] % range of a training epoch
130         if isinstance(self.val_check_interval, int):
131             self.val_check_batch = self.val_check_interval
132             if self.val_check_batch > self.num_training_batches:
133                 raise ValueError(
134                     f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '
135                     f'to the number of the training batches ({self.num_training_batches}). '
136                     'If you want to disable validation set `val_percent_check` to 0.0 instead.')
137         else:
138             if not _has_len(self.train_dataloader):
139                 raise MisconfigurationException(
140                     'When using an infinite DataLoader (e.g. with an IterableDataset or when '
141                     'DataLoader does not implement `__len__`) for `train_dataloader`, '
142                     '`Trainer(val_check_interval)` must be an int. An int k specifies checking '
143                     'validation every k training batches.')
144 
145             self._percent_range_check('val_check_interval')
146 
147             self.val_check_batch = int(self.num_training_batches * self.val_check_interval)
148             self.val_check_batch = max(1, self.val_check_batch)
149 
150     def _reset_eval_dataloader(self, model: LightningModule,
151                                mode: str) -> Tuple[int, List[DataLoader]]:
152         """Generic method to reset a dataloader for evaluation.
153 
154         Args:
155             model: The current `LightningModule`
156             mode: Either `'val'` or `'test'`
157 
158         Returns:
159             Tuple (num_batches, dataloaders)
160         """
161         dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))
162 
163         if not isinstance(dataloaders, list):
164             dataloaders = [dataloaders]
165 
166         # add samplers
167         dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]
168 
169         num_batches = 0
170 
171         # determine number of batches
172         # datasets could be none, 1 or 2+
173         if len(dataloaders) != 0:
174             for dataloader in dataloaders:
175                 if not _has_len(dataloader):
176                     num_batches = float('inf')
177                     break
178 
179             percent_check = getattr(self, f'{mode}_percent_check')
180 
181             if num_batches != float('inf'):
182                 self._percent_range_check(f'{mode}_percent_check')
183 
184                 num_batches = sum(len(dataloader) for dataloader in dataloaders)
185                 num_batches = int(num_batches * percent_check)
186             elif percent_check not in (0.0, 1.0):
187                 raise MisconfigurationException(
188                     'When using an infinite DataLoader (e.g. with an IterableDataset or when '
189                     f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '
190                     f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')
191         return num_batches, dataloaders
192 
193     def reset_val_dataloader(self, model: LightningModule) -> None:
194         """Resets the validation dataloader and determines the number of batches.
195 
196         Args:
197             model: The current `LightningModule`
198         """
199         if self.is_overriden('validation_step'):
200             self.num_val_batches, self.val_dataloaders =\
201                 self._reset_eval_dataloader(model, 'val')
202 
203     def reset_test_dataloader(self, model) -> None:
204         """Resets the validation dataloader and determines the number of batches.
205 
206         Args:
207             model: The current `LightningModule`
208         """
209         if self.is_overriden('test_step'):
210             self.num_test_batches, self.test_dataloaders =\
211                 self._reset_eval_dataloader(model, 'test')
212 
213     def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:
214         """Handles downloading data in the GPU or TPU case.
215 
216         Args:
217             dataloader_fx: The bound dataloader getter
218 
219         Returns:
220             The dataloader
221         """
222         dataloader = dataloader_fx()
223 
224         # get the function we'll use to get data
225         if self.use_ddp or self.use_ddp2:
226             # all processes wait until data download has happened
227             torch_distrib.barrier()
228 
229         # data download/load on TPU
230         elif self.use_tpu and XLA_AVAILABLE:
231             # all processes wait until data download has happened
232             torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')
233 
234         return dataloader
235 
236     def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,
237                                   test_percent_check: float, overfit_pct: float) -> None:
238         """Use less data for debugging purposes
239         """
240         self.train_percent_check = train_percent_check
241         self.val_percent_check = val_percent_check
242         self.test_percent_check = test_percent_check
243         if overfit_pct > 0:
244             if overfit_pct > 1:
245                 raise ValueError(
246                     f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')
247 
248             self.train_percent_check = overfit_pct
249             self.val_percent_check = overfit_pct
250             self.test_percent_check = overfit_pct
251
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -91,15 +91,12 @@
                 )
                 dl_args['shuffle'] = False
             else:
-                if train:
-                    sampler = DistributedSampler(dataloader.dataset)
-                    dl_args['shuffle'] = False
-                else:
-                    sampler = SequentialSampler(dataloader.dataset)
+                sampler = DistributedSampler(dataloader.dataset)
+                dl_args['shuffle'] = False
 
             dl_args['sampler'] = sampler
-
             dataloader = DataLoader(**dl_args)
+
         return dataloader
 
     def reset_train_dataloader(self, model: LightningModule) -> None:

{"golden_diff": "diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py\n--- a/pytorch_lightning/trainer/data_loading.py\n+++ b/pytorch_lightning/trainer/data_loading.py\n@@ -91,15 +91,12 @@\n                 )\n                 dl_args['shuffle'] = False\n             else:\n-                if train:\n-                    sampler = DistributedSampler(dataloader.dataset)\n-                    dl_args['shuffle'] = False\n-                else:\n-                    sampler = SequentialSampler(dataloader.dataset)\n+                sampler = DistributedSampler(dataloader.dataset)\n+                dl_args['shuffle'] = False\n \n             dl_args['sampler'] = sampler\n-\n             dataloader = DataLoader(**dl_args)\n+\n         return dataloader\n \n     def reset_train_dataloader(self, model: LightningModule) -> None:\n", "issue": "multi-gpu ddp calls validation and testing loops too many times\nWhen using ddp with multiple gpus, each validation and test loop is called with the entire validation dataset for each gpu.\r\n\r\nExpected behavior is that the dataset is divided appropriately across the gpus.\r\n\r\nI am using current master (cloned Mar 14), Ubuntu 19.10, Cuda 10.1, python 3.7.5, pytorch 1.4, venv environment.\r\n\r\nThe problem appears to be in `auto_add_sampler()` in data_loading.py. It does not create a `DistributedSampler` for validation or test datasets.\n", "before_files": [{"content": "from abc import ABC, abstractmethod\nfrom typing import Union, List, Tuple, Callable\n\nimport torch.distributed as torch_distrib\nfrom torch.utils.data import SequentialSampler, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities.debugging import MisconfigurationException\n\ntry:\n    from apex import amp\nexcept ImportError:\n    APEX_AVAILABLE = False\nelse:\n    APEX_AVAILABLE = True\n\ntry:\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.xla_multiprocessing as xmp\nexcept ImportError:\n    XLA_AVAILABLE = False\nelse:\n    XLA_AVAILABLE = True\n\n\ndef _has_len(dataloader: DataLoader) -> bool:\n    try:\n        # try getting the length\n        _ = len(dataloader)\n        return True\n    except TypeError:\n        return False\n\n\nclass TrainerDataLoadingMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    proc_rank: int\n    use_ddp: bool\n    use_ddp2: bool\n    shown_warnings: ...\n    val_check_interval: float\n    use_tpu: bool\n    tpu_local_core_rank: int\n    train_dataloader: DataLoader\n    num_training_batches: Union[int, float]\n    val_check_batch: ...\n    val_dataloaders: List[DataLoader]\n    num_val_batches: Union[int, float]\n    test_dataloaders: List[DataLoader]\n    num_test_batches: Union[int, float]\n    train_percent_check: float\n    val_percent_check: float\n    test_percent_check: float\n\n    @abstractmethod\n    def is_overriden(self, *args):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def _percent_range_check(self, name: str) -> None:\n        value = getattr(self, name)\n        msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'\n        if name == 'val_check_interval':\n            msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'\n\n        if not 0. <= value <= 1.:\n            raise ValueError(msg)\n\n    def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:\n        if self.use_ddp or self.use_ddp2 or self.use_tpu:\n            dl_args = {\n                'dataset': dataloader.dataset,\n                'batch_size': dataloader.batch_size,\n                'shuffle': False,\n                'num_workers': dataloader.num_workers,\n                'collate_fn': dataloader.collate_fn,\n                'pin_memory': dataloader.pin_memory,\n                'drop_last': dataloader.drop_last,\n                'timeout': dataloader.timeout,\n                'worker_init_fn': dataloader.worker_init_fn\n            }\n\n            if self.use_tpu:\n                sampler = DistributedSampler(\n                    dataloader.dataset,\n                    num_replicas=xm.xrt_world_size(),\n                    rank=xm.get_ordinal()\n                )\n                dl_args['shuffle'] = False\n            else:\n                if train:\n                    sampler = DistributedSampler(dataloader.dataset)\n                    dl_args['shuffle'] = False\n                else:\n                    sampler = SequentialSampler(dataloader.dataset)\n\n            dl_args['sampler'] = sampler\n\n            dataloader = DataLoader(**dl_args)\n        return dataloader\n\n    def reset_train_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the train dataloader and initialises required variables\n        (number of batches, when to validate, etc.).\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        self.train_dataloader = self.request_dataloader(model.train_dataloader)\n        self.num_training_batches = 0\n\n        # automatically add samplers\n        self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)\n\n        self._percent_range_check('train_percent_check')\n\n        if not _has_len(self.train_dataloader):\n            self.num_training_batches = float('inf')\n        else:\n            # try getting the length\n            self.num_training_batches = len(self.train_dataloader)\n            self.num_training_batches = int(self.num_training_batches * self.train_percent_check)\n\n        # determine when to check validation\n        # if int passed in, val checks that often\n        # otherwise, it checks in [0, 1.0] % range of a training epoch\n        if isinstance(self.val_check_interval, int):\n            self.val_check_batch = self.val_check_interval\n            if self.val_check_batch > self.num_training_batches:\n                raise ValueError(\n                    f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '\n                    f'to the number of the training batches ({self.num_training_batches}). '\n                    'If you want to disable validation set `val_percent_check` to 0.0 instead.')\n        else:\n            if not _has_len(self.train_dataloader):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    'DataLoader does not implement `__len__`) for `train_dataloader`, '\n                    '`Trainer(val_check_interval)` must be an int. An int k specifies checking '\n                    'validation every k training batches.')\n\n            self._percent_range_check('val_check_interval')\n\n            self.val_check_batch = int(self.num_training_batches * self.val_check_interval)\n            self.val_check_batch = max(1, self.val_check_batch)\n\n    def _reset_eval_dataloader(self, model: LightningModule,\n                               mode: str) -> Tuple[int, List[DataLoader]]:\n        \"\"\"Generic method to reset a dataloader for evaluation.\n\n        Args:\n            model: The current `LightningModule`\n            mode: Either `'val'` or `'test'`\n\n        Returns:\n            Tuple (num_batches, dataloaders)\n        \"\"\"\n        dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))\n\n        if not isinstance(dataloaders, list):\n            dataloaders = [dataloaders]\n\n        # add samplers\n        dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]\n\n        num_batches = 0\n\n        # determine number of batches\n        # datasets could be none, 1 or 2+\n        if len(dataloaders) != 0:\n            for dataloader in dataloaders:\n                if not _has_len(dataloader):\n                    num_batches = float('inf')\n                    break\n\n            percent_check = getattr(self, f'{mode}_percent_check')\n\n            if num_batches != float('inf'):\n                self._percent_range_check(f'{mode}_percent_check')\n\n                num_batches = sum(len(dataloader) for dataloader in dataloaders)\n                num_batches = int(num_batches * percent_check)\n            elif percent_check not in (0.0, 1.0):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '\n                    f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')\n        return num_batches, dataloaders\n\n    def reset_val_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('validation_step'):\n            self.num_val_batches, self.val_dataloaders =\\\n                self._reset_eval_dataloader(model, 'val')\n\n    def reset_test_dataloader(self, model) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('test_step'):\n            self.num_test_batches, self.test_dataloaders =\\\n                self._reset_eval_dataloader(model, 'test')\n\n    def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:\n        \"\"\"Handles downloading data in the GPU or TPU case.\n\n        Args:\n            dataloader_fx: The bound dataloader getter\n\n        Returns:\n            The dataloader\n        \"\"\"\n        dataloader = dataloader_fx()\n\n        # get the function we'll use to get data\n        if self.use_ddp or self.use_ddp2:\n            # all processes wait until data download has happened\n            torch_distrib.barrier()\n\n        # data download/load on TPU\n        elif self.use_tpu and XLA_AVAILABLE:\n            # all processes wait until data download has happened\n            torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')\n\n        return dataloader\n\n    def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,\n                                  test_percent_check: float, overfit_pct: float) -> None:\n        \"\"\"Use less data for debugging purposes\n        \"\"\"\n        self.train_percent_check = train_percent_check\n        self.val_percent_check = val_percent_check\n        self.test_percent_check = test_percent_check\n        if overfit_pct > 0:\n            if overfit_pct > 1:\n                raise ValueError(\n                    f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')\n\n            self.train_percent_check = overfit_pct\n            self.val_percent_check = overfit_pct\n            self.test_percent_check = overfit_pct\n", "path": "pytorch_lightning/trainer/data_loading.py"}], "after_files": [{"content": "from abc import ABC, abstractmethod\nfrom typing import Union, List, Tuple, Callable\n\nimport torch.distributed as torch_distrib\nfrom torch.utils.data import SequentialSampler, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities.debugging import MisconfigurationException\n\ntry:\n    from apex import amp\nexcept ImportError:\n    APEX_AVAILABLE = False\nelse:\n    APEX_AVAILABLE = True\n\ntry:\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.xla_multiprocessing as xmp\nexcept ImportError:\n    XLA_AVAILABLE = False\nelse:\n    XLA_AVAILABLE = True\n\n\ndef _has_len(dataloader: DataLoader) -> bool:\n    try:\n        # try getting the length\n        _ = len(dataloader)\n        return True\n    except TypeError:\n        return False\n\n\nclass TrainerDataLoadingMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    proc_rank: int\n    use_ddp: bool\n    use_ddp2: bool\n    shown_warnings: ...\n    val_check_interval: float\n    use_tpu: bool\n    tpu_local_core_rank: int\n    train_dataloader: DataLoader\n    num_training_batches: Union[int, float]\n    val_check_batch: ...\n    val_dataloaders: List[DataLoader]\n    num_val_batches: Union[int, float]\n    test_dataloaders: List[DataLoader]\n    num_test_batches: Union[int, float]\n    train_percent_check: float\n    val_percent_check: float\n    test_percent_check: float\n\n    @abstractmethod\n    def is_overriden(self, *args):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def _percent_range_check(self, name: str) -> None:\n        value = getattr(self, name)\n        msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'\n        if name == 'val_check_interval':\n            msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'\n\n        if not 0. <= value <= 1.:\n            raise ValueError(msg)\n\n    def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:\n        if self.use_ddp or self.use_ddp2 or self.use_tpu:\n            dl_args = {\n                'dataset': dataloader.dataset,\n                'batch_size': dataloader.batch_size,\n                'shuffle': False,\n                'num_workers': dataloader.num_workers,\n                'collate_fn': dataloader.collate_fn,\n                'pin_memory': dataloader.pin_memory,\n                'drop_last': dataloader.drop_last,\n                'timeout': dataloader.timeout,\n                'worker_init_fn': dataloader.worker_init_fn\n            }\n\n            if self.use_tpu:\n                sampler = DistributedSampler(\n                    dataloader.dataset,\n                    num_replicas=xm.xrt_world_size(),\n                    rank=xm.get_ordinal()\n                )\n                dl_args['shuffle'] = False\n            else:\n                sampler = DistributedSampler(dataloader.dataset)\n                dl_args['shuffle'] = False\n\n            dl_args['sampler'] = sampler\n            dataloader = DataLoader(**dl_args)\n\n        return dataloader\n\n    def reset_train_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the train dataloader and initialises required variables\n        (number of batches, when to validate, etc.).\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        self.train_dataloader = self.request_dataloader(model.train_dataloader)\n        self.num_training_batches = 0\n\n        # automatically add samplers\n        self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)\n\n        self._percent_range_check('train_percent_check')\n\n        if not _has_len(self.train_dataloader):\n            self.num_training_batches = float('inf')\n        else:\n            # try getting the length\n            self.num_training_batches = len(self.train_dataloader)\n            self.num_training_batches = int(self.num_training_batches * self.train_percent_check)\n\n        # determine when to check validation\n        # if int passed in, val checks that often\n        # otherwise, it checks in [0, 1.0] % range of a training epoch\n        if isinstance(self.val_check_interval, int):\n            self.val_check_batch = self.val_check_interval\n            if self.val_check_batch > self.num_training_batches:\n                raise ValueError(\n                    f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '\n                    f'to the number of the training batches ({self.num_training_batches}). '\n                    'If you want to disable validation set `val_percent_check` to 0.0 instead.')\n        else:\n            if not _has_len(self.train_dataloader):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    'DataLoader does not implement `__len__`) for `train_dataloader`, '\n                    '`Trainer(val_check_interval)` must be an int. An int k specifies checking '\n                    'validation every k training batches.')\n\n            self._percent_range_check('val_check_interval')\n\n            self.val_check_batch = int(self.num_training_batches * self.val_check_interval)\n            self.val_check_batch = max(1, self.val_check_batch)\n\n    def _reset_eval_dataloader(self, model: LightningModule,\n                               mode: str) -> Tuple[int, List[DataLoader]]:\n        \"\"\"Generic method to reset a dataloader for evaluation.\n\n        Args:\n            model: The current `LightningModule`\n            mode: Either `'val'` or `'test'`\n\n        Returns:\n            Tuple (num_batches, dataloaders)\n        \"\"\"\n        dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))\n\n        if not isinstance(dataloaders, list):\n            dataloaders = [dataloaders]\n\n        # add samplers\n        dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]\n\n        num_batches = 0\n\n        # determine number of batches\n        # datasets could be none, 1 or 2+\n        if len(dataloaders) != 0:\n            for dataloader in dataloaders:\n                if not _has_len(dataloader):\n                    num_batches = float('inf')\n                    break\n\n            percent_check = getattr(self, f'{mode}_percent_check')\n\n            if num_batches != float('inf'):\n                self._percent_range_check(f'{mode}_percent_check')\n\n                num_batches = sum(len(dataloader) for dataloader in dataloaders)\n                num_batches = int(num_batches * percent_check)\n            elif percent_check not in (0.0, 1.0):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '\n                    f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')\n        return num_batches, dataloaders\n\n    def reset_val_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('validation_step'):\n            self.num_val_batches, self.val_dataloaders =\\\n                self._reset_eval_dataloader(model, 'val')\n\n    def reset_test_dataloader(self, model) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('test_step'):\n            self.num_test_batches, self.test_dataloaders =\\\n                self._reset_eval_dataloader(model, 'test')\n\n    def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:\n        \"\"\"Handles downloading data in the GPU or TPU case.\n\n        Args:\n            dataloader_fx: The bound dataloader getter\n\n        Returns:\n            The dataloader\n        \"\"\"\n        dataloader = dataloader_fx()\n\n        # get the function we'll use to get data\n        if self.use_ddp or self.use_ddp2:\n            # all processes wait until data download has happened\n            torch_distrib.barrier()\n\n        # data download/load on TPU\n        elif self.use_tpu and XLA_AVAILABLE:\n            # all processes wait until data download has happened\n            torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')\n\n        return dataloader\n\n    def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,\n                                  test_percent_check: float, overfit_pct: float) -> None:\n        \"\"\"Use less data for debugging purposes\n        \"\"\"\n        self.train_percent_check = train_percent_check\n        self.val_percent_check = val_percent_check\n        self.test_percent_check = test_percent_check\n        if overfit_pct > 0:\n            if overfit_pct > 1:\n                raise ValueError(\n                    f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')\n\n            self.train_percent_check = overfit_pct\n            self.val_percent_check = overfit_pct\n            self.test_percent_check = overfit_pct\n", "path": "pytorch_lightning/trainer/data_loading.py"}]}