We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Training loop temporarily hangs after every 4 steps
I am porting some of my code to pytorch lightning, and everything seems to work fine. However, for some reason after every 4 training steps I see some temporary hanging (~1 second), which is severely slowing down my overall training time. Am I missing some obvious configuration?  This is my Trainer configuration:

```
    trainer = pl.Trainer(
        gpus=8
        num_nodes=1,
        distributed_backend='ddp',
        checkpoint_callback=False,
        max_epochs=50,
        max_steps=None,
        progress_bar_refresh_rate=1,
        check_val_every_n_epoch=1,
        val_check_interval=1.0,
        gradient_clip_val=0.0,
        log_save_interval=0,
        num_sanity_val_steps=0,
        amp_level='O0',
    )
```



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/trainer/data_loading.py`
Content:
```
1 from abc import ABC, abstractmethod
2 from typing import Union, List, Tuple, Callable
3 
4 import torch.distributed as torch_distrib
5 from torch.utils.data import SequentialSampler, DataLoader
6 from torch.utils.data.distributed import DistributedSampler
7 
8 from pytorch_lightning.core import LightningModule
9 from pytorch_lightning.utilities.exceptions import MisconfigurationException
10 
11 try:
12     from apex import amp
13 except ImportError:
14     APEX_AVAILABLE = False
15 else:
16     APEX_AVAILABLE = True
17 
18 try:
19     import torch_xla
20     import torch_xla.core.xla_model as xm
21     import torch_xla.distributed.xla_multiprocessing as xmp
22 except ImportError:
23     XLA_AVAILABLE = False
24 else:
25     XLA_AVAILABLE = True
26 
27 
28 def _has_len(dataloader: DataLoader) -> bool:
29     """ Checks if a given Dataloader has __len__ method implemented i.e. if
30     it is a finite dataloader or infinite dataloader """
31     try:
32         # try getting the length
33         if len(dataloader) == 0:
34             raise ValueError('Dataloader returned 0 length. Please make sure'
35                              ' that your Dataloader atleast returns 1 batch')
36         return True
37     except TypeError:
38         return False
39 
40 
41 class TrainerDataLoadingMixin(ABC):
42 
43     # this is just a summary on variables used in this abstract class,
44     #  the proper values/initialisation should be done in child class
45     proc_rank: int
46     use_ddp: bool
47     use_ddp2: bool
48     shown_warnings: ...
49     val_check_interval: float
50     use_tpu: bool
51     tpu_local_core_rank: int
52     train_dataloader: DataLoader
53     num_training_batches: Union[int, float]
54     val_check_batch: ...
55     val_dataloaders: List[DataLoader]
56     num_val_batches: Union[int, float]
57     test_dataloaders: List[DataLoader]
58     num_test_batches: Union[int, float]
59     train_percent_check: float
60     val_percent_check: float
61     test_percent_check: float
62 
63     @abstractmethod
64     def is_overriden(self, *args):
65         """Warning: this is just empty shell for code implemented in other class."""
66 
67     def _percent_range_check(self, name: str) -> None:
68         value = getattr(self, name)
69         msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'
70         if name == 'val_check_interval':
71             msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'
72 
73         if not 0. <= value <= 1.:
74             raise ValueError(msg)
75 
76     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
77 
78         # don't do anything if it's not a dataloader
79         if not isinstance(dataloader, DataLoader):
80             return dataloader
81 
82         need_dist_sampler = self.use_ddp or self.use_ddp2 or self.use_tpu
83         no_sampler_added = dataloader.sampler is None
84 
85         if need_dist_sampler and no_sampler_added:
86 
87             skip_keys = ['sampler', 'batch_sampler', 'dataset_kind']
88 
89             dl_args = {
90                 k: v for k, v in dataloader.__dict__.items() if not k.startswith('_') and k not in skip_keys
91             }
92 
93             if self.use_tpu:
94                 sampler = DistributedSampler(
95                     dataloader.dataset,
96                     num_replicas=xm.xrt_world_size(),
97                     rank=xm.get_ordinal()
98                 )
99             else:
100                 sampler = DistributedSampler(dataloader.dataset)
101 
102             dl_args['sampler'] = sampler
103             dataloader = type(dataloader)(**dl_args)
104 
105         return dataloader
106 
107     def reset_train_dataloader(self, model: LightningModule) -> None:
108         """Resets the train dataloader and initialises required variables
109         (number of batches, when to validate, etc.).
110 
111         Args:
112             model: The current `LightningModule`
113         """
114         self.train_dataloader = self.request_dataloader(model.train_dataloader)
115         self.num_training_batches = 0
116 
117         # automatically add samplers
118         self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)
119 
120         self._percent_range_check('train_percent_check')
121 
122         if not _has_len(self.train_dataloader):
123             self.num_training_batches = float('inf')
124         else:
125             # try getting the length
126             self.num_training_batches = len(self.train_dataloader)
127             self.num_training_batches = int(self.num_training_batches * self.train_percent_check)
128 
129         # determine when to check validation
130         # if int passed in, val checks that often
131         # otherwise, it checks in [0, 1.0] % range of a training epoch
132         if isinstance(self.val_check_interval, int):
133             self.val_check_batch = self.val_check_interval
134             if self.val_check_batch > self.num_training_batches:
135                 raise ValueError(
136                     f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '
137                     f'to the number of the training batches ({self.num_training_batches}). '
138                     'If you want to disable validation set `val_percent_check` to 0.0 instead.')
139         else:
140             if not _has_len(self.train_dataloader):
141                 if self.val_check_interval == 1.0:
142                     self.val_check_batch = float('inf')
143                 else:
144                     raise MisconfigurationException(
145                         'When using an infinite DataLoader (e.g. with an IterableDataset or when '
146                         'DataLoader does not implement `__len__`) for `train_dataloader`, '
147                         '`Trainer(val_check_interval)` must be `1.0` or an int. An int k specifies '
148                         'checking validation every k training batches.')
149             else:
150                 self._percent_range_check('val_check_interval')
151 
152                 self.val_check_batch = int(self.num_training_batches * self.val_check_interval)
153                 self.val_check_batch = max(1, self.val_check_batch)
154 
155     def _reset_eval_dataloader(self, model: LightningModule,
156                                mode: str) -> Tuple[int, List[DataLoader]]:
157         """Generic method to reset a dataloader for evaluation.
158 
159         Args:
160             model: The current `LightningModule`
161             mode: Either `'val'` or `'test'`
162 
163         Returns:
164             Tuple (num_batches, dataloaders)
165         """
166         dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))
167 
168         if not isinstance(dataloaders, list):
169             dataloaders = [dataloaders]
170 
171         # add samplers
172         dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]
173 
174         num_batches = 0
175 
176         # determine number of batches
177         # datasets could be none, 1 or 2+
178         if len(dataloaders) != 0:
179             for dataloader in dataloaders:
180                 if not _has_len(dataloader):
181                     num_batches = float('inf')
182                     break
183 
184             percent_check = getattr(self, f'{mode}_percent_check')
185 
186             if num_batches != float('inf'):
187                 self._percent_range_check(f'{mode}_percent_check')
188 
189                 num_batches = sum(len(dataloader) for dataloader in dataloaders)
190                 num_batches = int(num_batches * percent_check)
191             elif percent_check not in (0.0, 1.0):
192                 raise MisconfigurationException(
193                     'When using an infinite DataLoader (e.g. with an IterableDataset or when '
194                     f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '
195                     f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')
196         return num_batches, dataloaders
197 
198     def reset_val_dataloader(self, model: LightningModule) -> None:
199         """Resets the validation dataloader and determines the number of batches.
200 
201         Args:
202             model: The current `LightningModule`
203         """
204         if self.is_overriden('validation_step'):
205             self.num_val_batches, self.val_dataloaders =\
206                 self._reset_eval_dataloader(model, 'val')
207 
208     def reset_test_dataloader(self, model) -> None:
209         """Resets the validation dataloader and determines the number of batches.
210 
211         Args:
212             model: The current `LightningModule`
213         """
214         if self.is_overriden('test_step'):
215             self.num_test_batches, self.test_dataloaders =\
216                 self._reset_eval_dataloader(model, 'test')
217 
218     def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:
219         """Handles downloading data in the GPU or TPU case.
220 
221         Args:
222             dataloader_fx: The bound dataloader getter
223 
224         Returns:
225             The dataloader
226         """
227         dataloader = dataloader_fx()
228 
229         # get the function we'll use to get data
230         if self.use_ddp or self.use_ddp2:
231             # all processes wait until data download has happened
232             torch_distrib.barrier()
233 
234         # data download/load on TPU
235         elif self.use_tpu and XLA_AVAILABLE:
236             # all processes wait until data download has happened
237             torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')
238 
239         return dataloader
240 
241     def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,
242                                   test_percent_check: float, overfit_pct: float) -> None:
243         """Use less data for debugging purposes
244         """
245         self.train_percent_check = train_percent_check
246         self.val_percent_check = val_percent_check
247         self.test_percent_check = test_percent_check
248         if overfit_pct > 0:
249             if overfit_pct > 1:
250                 raise ValueError(
251                     f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')
252 
253             self.train_percent_check = overfit_pct
254             self.val_percent_check = overfit_pct
255             self.test_percent_check = overfit_pct
256
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -1,8 +1,9 @@
+import warnings
 from abc import ABC, abstractmethod
 from typing import Union, List, Tuple, Callable
 
 import torch.distributed as torch_distrib
-from torch.utils.data import SequentialSampler, DataLoader
+from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
 from pytorch_lightning.core import LightningModule
@@ -73,6 +74,12 @@
         if not 0. <= value <= 1.:
             raise ValueError(msg)
 
+    def _worker_check(self, dataloader: DataLoader, name: str) -> None:
+        if isinstance(dataloader, DataLoader) and dataloader.num_workers <= 2:
+            warnings.warn(f'The dataloader, {name}, does not have many workers which may be a bottleneck.'
+                          ' Consider increasing the value of the `num_workers` argument`'
+                          ' in the `DataLoader` init to improve performance.')
+
     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
 
         # don't do anything if it's not a dataloader
@@ -112,11 +119,13 @@
             model: The current `LightningModule`
         """
         self.train_dataloader = self.request_dataloader(model.train_dataloader)
+
         self.num_training_batches = 0
 
         # automatically add samplers
         self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)
 
+        self._worker_check(self.train_dataloader, 'train dataloader')
         self._percent_range_check('train_percent_check')
 
         if not _has_len(self.train_dataloader):
@@ -176,10 +185,10 @@
         # determine number of batches
         # datasets could be none, 1 or 2+
         if len(dataloaders) != 0:
-            for dataloader in dataloaders:
+            for i, dataloader in enumerate(dataloaders):
+                self._worker_check(dataloader, f'{mode} dataloader {i}')
                 if not _has_len(dataloader):
                     num_batches = float('inf')
-                    break
 
             percent_check = getattr(self, f'{mode}_percent_check')

{"golden_diff": "diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py\n--- a/pytorch_lightning/trainer/data_loading.py\n+++ b/pytorch_lightning/trainer/data_loading.py\n@@ -1,8 +1,9 @@\n+import warnings\n from abc import ABC, abstractmethod\n from typing import Union, List, Tuple, Callable\n \n import torch.distributed as torch_distrib\n-from torch.utils.data import SequentialSampler, DataLoader\n+from torch.utils.data import DataLoader\n from torch.utils.data.distributed import DistributedSampler\n \n from pytorch_lightning.core import LightningModule\n@@ -73,6 +74,12 @@\n         if not 0. <= value <= 1.:\n             raise ValueError(msg)\n \n+    def _worker_check(self, dataloader: DataLoader, name: str) -> None:\n+        if isinstance(dataloader, DataLoader) and dataloader.num_workers <= 2:\n+            warnings.warn(f'The dataloader, {name}, does not have many workers which may be a bottleneck.'\n+                          ' Consider increasing the value of the `num_workers` argument`'\n+                          ' in the `DataLoader` init to improve performance.')\n+\n     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:\n \n         # don't do anything if it's not a dataloader\n@@ -112,11 +119,13 @@\n             model: The current `LightningModule`\n         \"\"\"\n         self.train_dataloader = self.request_dataloader(model.train_dataloader)\n+\n         self.num_training_batches = 0\n \n         # automatically add samplers\n         self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)\n \n+        self._worker_check(self.train_dataloader, 'train dataloader')\n         self._percent_range_check('train_percent_check')\n \n         if not _has_len(self.train_dataloader):\n@@ -176,10 +185,10 @@\n         # determine number of batches\n         # datasets could be none, 1 or 2+\n         if len(dataloaders) != 0:\n-            for dataloader in dataloaders:\n+            for i, dataloader in enumerate(dataloaders):\n+                self._worker_check(dataloader, f'{mode} dataloader {i}')\n                 if not _has_len(dataloader):\n                     num_batches = float('inf')\n-                    break\n \n             percent_check = getattr(self, f'{mode}_percent_check')\n", "issue": "Training loop temporarily hangs after every 4 steps\nI am porting some of my code to pytorch lightning, and everything seems to work fine. However, for some reason after every 4 training steps I see some temporary hanging (~1 second), which is severely slowing down my overall training time. Am I missing some obvious configuration?  This is my Trainer configuration:\r\n\r\n```\r\n    trainer = pl.Trainer(\r\n        gpus=8\r\n        num_nodes=1,\r\n        distributed_backend='ddp',\r\n        checkpoint_callback=False,\r\n        max_epochs=50,\r\n        max_steps=None,\r\n        progress_bar_refresh_rate=1,\r\n        check_val_every_n_epoch=1,\r\n        val_check_interval=1.0,\r\n        gradient_clip_val=0.0,\r\n        log_save_interval=0,\r\n        num_sanity_val_steps=0,\r\n        amp_level='O0',\r\n    )\r\n```\r\n\r\n\n", "before_files": [{"content": "from abc import ABC, abstractmethod\nfrom typing import Union, List, Tuple, Callable\n\nimport torch.distributed as torch_distrib\nfrom torch.utils.data import SequentialSampler, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities.exceptions import MisconfigurationException\n\ntry:\n    from apex import amp\nexcept ImportError:\n    APEX_AVAILABLE = False\nelse:\n    APEX_AVAILABLE = True\n\ntry:\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.xla_multiprocessing as xmp\nexcept ImportError:\n    XLA_AVAILABLE = False\nelse:\n    XLA_AVAILABLE = True\n\n\ndef _has_len(dataloader: DataLoader) -> bool:\n    \"\"\" Checks if a given Dataloader has __len__ method implemented i.e. if\n    it is a finite dataloader or infinite dataloader \"\"\"\n    try:\n        # try getting the length\n        if len(dataloader) == 0:\n            raise ValueError('Dataloader returned 0 length. Please make sure'\n                             ' that your Dataloader atleast returns 1 batch')\n        return True\n    except TypeError:\n        return False\n\n\nclass TrainerDataLoadingMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    proc_rank: int\n    use_ddp: bool\n    use_ddp2: bool\n    shown_warnings: ...\n    val_check_interval: float\n    use_tpu: bool\n    tpu_local_core_rank: int\n    train_dataloader: DataLoader\n    num_training_batches: Union[int, float]\n    val_check_batch: ...\n    val_dataloaders: List[DataLoader]\n    num_val_batches: Union[int, float]\n    test_dataloaders: List[DataLoader]\n    num_test_batches: Union[int, float]\n    train_percent_check: float\n    val_percent_check: float\n    test_percent_check: float\n\n    @abstractmethod\n    def is_overriden(self, *args):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def _percent_range_check(self, name: str) -> None:\n        value = getattr(self, name)\n        msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'\n        if name == 'val_check_interval':\n            msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'\n\n        if not 0. <= value <= 1.:\n            raise ValueError(msg)\n\n    def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:\n\n        # don't do anything if it's not a dataloader\n        if not isinstance(dataloader, DataLoader):\n            return dataloader\n\n        need_dist_sampler = self.use_ddp or self.use_ddp2 or self.use_tpu\n        no_sampler_added = dataloader.sampler is None\n\n        if need_dist_sampler and no_sampler_added:\n\n            skip_keys = ['sampler', 'batch_sampler', 'dataset_kind']\n\n            dl_args = {\n                k: v for k, v in dataloader.__dict__.items() if not k.startswith('_') and k not in skip_keys\n            }\n\n            if self.use_tpu:\n                sampler = DistributedSampler(\n                    dataloader.dataset,\n                    num_replicas=xm.xrt_world_size(),\n                    rank=xm.get_ordinal()\n                )\n            else:\n                sampler = DistributedSampler(dataloader.dataset)\n\n            dl_args['sampler'] = sampler\n            dataloader = type(dataloader)(**dl_args)\n\n        return dataloader\n\n    def reset_train_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the train dataloader and initialises required variables\n        (number of batches, when to validate, etc.).\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        self.train_dataloader = self.request_dataloader(model.train_dataloader)\n        self.num_training_batches = 0\n\n        # automatically add samplers\n        self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)\n\n        self._percent_range_check('train_percent_check')\n\n        if not _has_len(self.train_dataloader):\n            self.num_training_batches = float('inf')\n        else:\n            # try getting the length\n            self.num_training_batches = len(self.train_dataloader)\n            self.num_training_batches = int(self.num_training_batches * self.train_percent_check)\n\n        # determine when to check validation\n        # if int passed in, val checks that often\n        # otherwise, it checks in [0, 1.0] % range of a training epoch\n        if isinstance(self.val_check_interval, int):\n            self.val_check_batch = self.val_check_interval\n            if self.val_check_batch > self.num_training_batches:\n                raise ValueError(\n                    f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '\n                    f'to the number of the training batches ({self.num_training_batches}). '\n                    'If you want to disable validation set `val_percent_check` to 0.0 instead.')\n        else:\n            if not _has_len(self.train_dataloader):\n                if self.val_check_interval == 1.0:\n                    self.val_check_batch = float('inf')\n                else:\n                    raise MisconfigurationException(\n                        'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                        'DataLoader does not implement `__len__`) for `train_dataloader`, '\n                        '`Trainer(val_check_interval)` must be `1.0` or an int. An int k specifies '\n                        'checking validation every k training batches.')\n            else:\n                self._percent_range_check('val_check_interval')\n\n                self.val_check_batch = int(self.num_training_batches * self.val_check_interval)\n                self.val_check_batch = max(1, self.val_check_batch)\n\n    def _reset_eval_dataloader(self, model: LightningModule,\n                               mode: str) -> Tuple[int, List[DataLoader]]:\n        \"\"\"Generic method to reset a dataloader for evaluation.\n\n        Args:\n            model: The current `LightningModule`\n            mode: Either `'val'` or `'test'`\n\n        Returns:\n            Tuple (num_batches, dataloaders)\n        \"\"\"\n        dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))\n\n        if not isinstance(dataloaders, list):\n            dataloaders = [dataloaders]\n\n        # add samplers\n        dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]\n\n        num_batches = 0\n\n        # determine number of batches\n        # datasets could be none, 1 or 2+\n        if len(dataloaders) != 0:\n            for dataloader in dataloaders:\n                if not _has_len(dataloader):\n                    num_batches = float('inf')\n                    break\n\n            percent_check = getattr(self, f'{mode}_percent_check')\n\n            if num_batches != float('inf'):\n                self._percent_range_check(f'{mode}_percent_check')\n\n                num_batches = sum(len(dataloader) for dataloader in dataloaders)\n                num_batches = int(num_batches * percent_check)\n            elif percent_check not in (0.0, 1.0):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '\n                    f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')\n        return num_batches, dataloaders\n\n    def reset_val_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('validation_step'):\n            self.num_val_batches, self.val_dataloaders =\\\n                self._reset_eval_dataloader(model, 'val')\n\n    def reset_test_dataloader(self, model) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('test_step'):\n            self.num_test_batches, self.test_dataloaders =\\\n                self._reset_eval_dataloader(model, 'test')\n\n    def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:\n        \"\"\"Handles downloading data in the GPU or TPU case.\n\n        Args:\n            dataloader_fx: The bound dataloader getter\n\n        Returns:\n            The dataloader\n        \"\"\"\n        dataloader = dataloader_fx()\n\n        # get the function we'll use to get data\n        if self.use_ddp or self.use_ddp2:\n            # all processes wait until data download has happened\n            torch_distrib.barrier()\n\n        # data download/load on TPU\n        elif self.use_tpu and XLA_AVAILABLE:\n            # all processes wait until data download has happened\n            torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')\n\n        return dataloader\n\n    def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,\n                                  test_percent_check: float, overfit_pct: float) -> None:\n        \"\"\"Use less data for debugging purposes\n        \"\"\"\n        self.train_percent_check = train_percent_check\n        self.val_percent_check = val_percent_check\n        self.test_percent_check = test_percent_check\n        if overfit_pct > 0:\n            if overfit_pct > 1:\n                raise ValueError(\n                    f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')\n\n            self.train_percent_check = overfit_pct\n            self.val_percent_check = overfit_pct\n            self.test_percent_check = overfit_pct\n", "path": "pytorch_lightning/trainer/data_loading.py"}], "after_files": [{"content": "import warnings\nfrom abc import ABC, abstractmethod\nfrom typing import Union, List, Tuple, Callable\n\nimport torch.distributed as torch_distrib\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\n\nfrom pytorch_lightning.core import LightningModule\nfrom pytorch_lightning.utilities.exceptions import MisconfigurationException\n\ntry:\n    from apex import amp\nexcept ImportError:\n    APEX_AVAILABLE = False\nelse:\n    APEX_AVAILABLE = True\n\ntry:\n    import torch_xla\n    import torch_xla.core.xla_model as xm\n    import torch_xla.distributed.xla_multiprocessing as xmp\nexcept ImportError:\n    XLA_AVAILABLE = False\nelse:\n    XLA_AVAILABLE = True\n\n\ndef _has_len(dataloader: DataLoader) -> bool:\n    \"\"\" Checks if a given Dataloader has __len__ method implemented i.e. if\n    it is a finite dataloader or infinite dataloader \"\"\"\n    try:\n        # try getting the length\n        if len(dataloader) == 0:\n            raise ValueError('Dataloader returned 0 length. Please make sure'\n                             ' that your Dataloader atleast returns 1 batch')\n        return True\n    except TypeError:\n        return False\n\n\nclass TrainerDataLoadingMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    proc_rank: int\n    use_ddp: bool\n    use_ddp2: bool\n    shown_warnings: ...\n    val_check_interval: float\n    use_tpu: bool\n    tpu_local_core_rank: int\n    train_dataloader: DataLoader\n    num_training_batches: Union[int, float]\n    val_check_batch: ...\n    val_dataloaders: List[DataLoader]\n    num_val_batches: Union[int, float]\n    test_dataloaders: List[DataLoader]\n    num_test_batches: Union[int, float]\n    train_percent_check: float\n    val_percent_check: float\n    test_percent_check: float\n\n    @abstractmethod\n    def is_overriden(self, *args):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def _percent_range_check(self, name: str) -> None:\n        value = getattr(self, name)\n        msg = f'`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}.'\n        if name == 'val_check_interval':\n            msg += ' If you want to disable validation set `val_percent_check` to 0.0 instead.'\n\n        if not 0. <= value <= 1.:\n            raise ValueError(msg)\n\n    def _worker_check(self, dataloader: DataLoader, name: str) -> None:\n        if isinstance(dataloader, DataLoader) and dataloader.num_workers <= 2:\n            warnings.warn(f'The dataloader, {name}, does not have many workers which may be a bottleneck.'\n                          ' Consider increasing the value of the `num_workers` argument`'\n                          ' in the `DataLoader` init to improve performance.')\n\n    def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:\n\n        # don't do anything if it's not a dataloader\n        if not isinstance(dataloader, DataLoader):\n            return dataloader\n\n        need_dist_sampler = self.use_ddp or self.use_ddp2 or self.use_tpu\n        no_sampler_added = dataloader.sampler is None\n\n        if need_dist_sampler and no_sampler_added:\n\n            skip_keys = ['sampler', 'batch_sampler', 'dataset_kind']\n\n            dl_args = {\n                k: v for k, v in dataloader.__dict__.items() if not k.startswith('_') and k not in skip_keys\n            }\n\n            if self.use_tpu:\n                sampler = DistributedSampler(\n                    dataloader.dataset,\n                    num_replicas=xm.xrt_world_size(),\n                    rank=xm.get_ordinal()\n                )\n            else:\n                sampler = DistributedSampler(dataloader.dataset)\n\n            dl_args['sampler'] = sampler\n            dataloader = type(dataloader)(**dl_args)\n\n        return dataloader\n\n    def reset_train_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the train dataloader and initialises required variables\n        (number of batches, when to validate, etc.).\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        self.train_dataloader = self.request_dataloader(model.train_dataloader)\n\n        self.num_training_batches = 0\n\n        # automatically add samplers\n        self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)\n\n        self._worker_check(self.train_dataloader, 'train dataloader')\n        self._percent_range_check('train_percent_check')\n\n        if not _has_len(self.train_dataloader):\n            self.num_training_batches = float('inf')\n        else:\n            # try getting the length\n            self.num_training_batches = len(self.train_dataloader)\n            self.num_training_batches = int(self.num_training_batches * self.train_percent_check)\n\n        # determine when to check validation\n        # if int passed in, val checks that often\n        # otherwise, it checks in [0, 1.0] % range of a training epoch\n        if isinstance(self.val_check_interval, int):\n            self.val_check_batch = self.val_check_interval\n            if self.val_check_batch > self.num_training_batches:\n                raise ValueError(\n                    f'`val_check_interval` ({self.val_check_interval}) must be less than or equal '\n                    f'to the number of the training batches ({self.num_training_batches}). '\n                    'If you want to disable validation set `val_percent_check` to 0.0 instead.')\n        else:\n            if not _has_len(self.train_dataloader):\n                if self.val_check_interval == 1.0:\n                    self.val_check_batch = float('inf')\n                else:\n                    raise MisconfigurationException(\n                        'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                        'DataLoader does not implement `__len__`) for `train_dataloader`, '\n                        '`Trainer(val_check_interval)` must be `1.0` or an int. An int k specifies '\n                        'checking validation every k training batches.')\n            else:\n                self._percent_range_check('val_check_interval')\n\n                self.val_check_batch = int(self.num_training_batches * self.val_check_interval)\n                self.val_check_batch = max(1, self.val_check_batch)\n\n    def _reset_eval_dataloader(self, model: LightningModule,\n                               mode: str) -> Tuple[int, List[DataLoader]]:\n        \"\"\"Generic method to reset a dataloader for evaluation.\n\n        Args:\n            model: The current `LightningModule`\n            mode: Either `'val'` or `'test'`\n\n        Returns:\n            Tuple (num_batches, dataloaders)\n        \"\"\"\n        dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))\n\n        if not isinstance(dataloaders, list):\n            dataloaders = [dataloaders]\n\n        # add samplers\n        dataloaders = [self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl]\n\n        num_batches = 0\n\n        # determine number of batches\n        # datasets could be none, 1 or 2+\n        if len(dataloaders) != 0:\n            for i, dataloader in enumerate(dataloaders):\n                self._worker_check(dataloader, f'{mode} dataloader {i}')\n                if not _has_len(dataloader):\n                    num_batches = float('inf')\n\n            percent_check = getattr(self, f'{mode}_percent_check')\n\n            if num_batches != float('inf'):\n                self._percent_range_check(f'{mode}_percent_check')\n\n                num_batches = sum(len(dataloader) for dataloader in dataloaders)\n                num_batches = int(num_batches * percent_check)\n            elif percent_check not in (0.0, 1.0):\n                raise MisconfigurationException(\n                    'When using an infinite DataLoader (e.g. with an IterableDataset or when '\n                    f'DataLoader does not implement `__len__`) for `{mode}_dataloader`, '\n                    f'`Trainer({mode}_percent_check)` must be `0.0` or `1.0`.')\n        return num_batches, dataloaders\n\n    def reset_val_dataloader(self, model: LightningModule) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('validation_step'):\n            self.num_val_batches, self.val_dataloaders =\\\n                self._reset_eval_dataloader(model, 'val')\n\n    def reset_test_dataloader(self, model) -> None:\n        \"\"\"Resets the validation dataloader and determines the number of batches.\n\n        Args:\n            model: The current `LightningModule`\n        \"\"\"\n        if self.is_overriden('test_step'):\n            self.num_test_batches, self.test_dataloaders =\\\n                self._reset_eval_dataloader(model, 'test')\n\n    def request_dataloader(self, dataloader_fx: Callable) -> DataLoader:\n        \"\"\"Handles downloading data in the GPU or TPU case.\n\n        Args:\n            dataloader_fx: The bound dataloader getter\n\n        Returns:\n            The dataloader\n        \"\"\"\n        dataloader = dataloader_fx()\n\n        # get the function we'll use to get data\n        if self.use_ddp or self.use_ddp2:\n            # all processes wait until data download has happened\n            torch_distrib.barrier()\n\n        # data download/load on TPU\n        elif self.use_tpu and XLA_AVAILABLE:\n            # all processes wait until data download has happened\n            torch_xla.core.xla_model.rendezvous('pl.TrainerDataLoadingMixin.get_dataloaders')\n\n        return dataloader\n\n    def determine_data_use_amount(self, train_percent_check: float, val_percent_check: float,\n                                  test_percent_check: float, overfit_pct: float) -> None:\n        \"\"\"Use less data for debugging purposes\n        \"\"\"\n        self.train_percent_check = train_percent_check\n        self.val_percent_check = val_percent_check\n        self.test_percent_check = test_percent_check\n        if overfit_pct > 0:\n            if overfit_pct > 1:\n                raise ValueError(\n                    f'`overfit_pct` must be not greater than 1.0, but got {overfit_pct:.3f}.')\n\n            self.train_percent_check = overfit_pct\n            self.val_percent_check = overfit_pct\n            self.test_percent_check = overfit_pct\n", "path": "pytorch_lightning/trainer/data_loading.py"}]}