We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Incorrect total number of batches
**Describe the bug**
Sometimes total number of batches is computed wrong.

**To Reproduce**
Run the following code with current master branch:
```
from time import sleep
import torch
from torch.utils.data import DataLoader, Dataset

import pytorch_lightning as pl


class DummyDataset(Dataset):
    def __init__(self, n):
        super().__init__()
        self.n = n

    def __len__(self):
        return self.n

    def __getitem__(self, idx):
        return torch.rand(10)


class CoolSystem(pl.LightningModule):
    def __init__(self):
        super(CoolSystem, self).__init__()
        self.layer = torch.nn.Linear(10, 10)

    def forward(self, x):
        return self.layer(x)

    def training_step(self, batch, batch_nb):
        sleep(1)
        return {'loss': torch.mean(self.forward(batch) ** 2)}

    def validation_step(self, batch, batch_nb):
        sleep(1)
        return {}

    def validation_end(self, outputs):
        return {}

    def configure_optimizers(self):
        return [torch.optim.Adam(self.layer.parameters())]

    @pl.data_loader
    def train_dataloader(self):
        return DataLoader(DummyDataset(10), batch_size=1)

    @pl.data_loader
    def val_dataloader(self):
        return DataLoader(DummyDataset(5), batch_size=1)

model = CoolSystem()
trainer = pl.Trainer(weights_summary=None, nb_sanity_val_steps=0, early_stop_callback=False,
                     val_percent_check=1.0, val_check_interval=0.5)
trainer.fit(model)
```
At first output will look like:
`67%|█████▋                    | 10/15 [00:10<00:05,  1.05s/it, batch_nb=4, epoch=0, loss=0.194, v_nb=0]`

But at the end of the epoch it will be like:
`20it [00:20,  1.07s/it, batch_nb=9, epoch=0, loss=0.212, v_nb=0]`

Moreover, if you run
```
trainer = pl.Trainer(weights_summary=None, nb_sanity_val_steps=0, early_stop_callback=False,
                     val_percent_check=1.0, val_check_interval=0.5, check_val_every_n_epoch=10)
```
The first epoch will end at the point 
`67%|█████▋                    | 10/15 [00:09<00:04,  1.01it/s, batch_nb=8, epoch=1, loss=0.069, v_nb=0]` 

**Expected behavior**
Correct total number of batches.

**Possible solution**
Now we have `total_batches = nb_training_batches + nb_val_batches`, where `nb_val_batches` is the number of batches of only one validation loop. And the problem arises because actually there can be several validation loops during one training epoch. Moreover there is a parameter `check_val_every_n_epoch` and thus there can be no validation loops at all.

With this in mind, it looks like the correct formula is:
```
is_val_epoch = (current_epoch + 1) % check_val_every_n_epoch == 0
val_checks_per_epoch = nb_training_batches // val_check_batch if is_val_epoch else 0
total_batches = nb_training_batches + nb_val_batches * val_checks_per_epoch
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `pytorch_lightning/trainer/train_loop_mixin.py`
Content:
```
1 import numpy as np
2 
3 try:
4     from apex import amp
5 
6     APEX_AVAILABLE = True
7 except ImportError:
8     APEX_AVAILABLE = False
9 
10 
11 class TrainerTrainLoopMixin(object):
12 
13     def train(self):
14         # run all epochs
15         for epoch_nb in range(self.current_epoch, self.max_nb_epochs):
16             # set seed for distributed sampler (enables shuffling for each epoch)
17             if self.use_ddp and hasattr(self.get_train_dataloader().sampler, 'set_epoch'):
18                 self.get_train_dataloader().sampler.set_epoch(epoch_nb)
19 
20             # get model
21             model = self.get_model()
22 
23             # update training progress in trainer and model
24             model.current_epoch = epoch_nb
25             self.current_epoch = epoch_nb
26             self.total_batches = self.nb_training_batches + self.nb_val_batches
27             self.batch_loss_value = 0  # accumulated grads
28 
29             # limit the number of batches to 1 in fast_dev_run
30             if self.fast_dev_run:
31                 self.total_batches = 1
32 
33             # init progress_bar when requested
34             if self.show_progress_bar:
35                 nb_iterations = self.total_batches
36 
37                 #  for iterable train loader, the progress bar never ends
38                 if self.is_iterable_train_dataloader:
39                     nb_iterations = float('inf')
40                 self.progress_bar.reset(nb_iterations)
41 
42             # changing gradient according accumulation_scheduler
43             self.accumulation_scheduler.on_epoch_begin(epoch_nb, self)
44 
45             # -----------------
46             # RUN TNG EPOCH
47             # -----------------
48             self.run_training_epoch()
49 
50             # update LR schedulers
51             if self.lr_schedulers is not None:
52                 for lr_scheduler in self.lr_schedulers:
53                     lr_scheduler.step(self.current_epoch)
54 
55             # early stopping
56             met_min_epochs = epoch_nb > self.min_nb_epochs
57             if self.enable_early_stop and (met_min_epochs or self.fast_dev_run):
58                 should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb,
59                                                                     logs=self.callback_metrics)
60                 # stop training
61                 stop = should_stop and met_min_epochs
62                 if stop:
63                     return
64 
65         if self.logger is not None:
66             self.logger.finalize("success")
67 
68     def run_training_epoch(self):
69         # before epoch hook
70         if self.is_function_implemented('on_epoch_start'):
71             model = self.get_model()
72             model.on_epoch_start()
73 
74         # run epoch
75         for batch_nb, batch in enumerate(self.get_train_dataloader()):
76             self.batch_nb = batch_nb
77 
78             model = self.get_model()
79             model.global_step = self.global_step
80 
81             # ---------------
82             # RUN TRAIN STEP
83             # ---------------
84             output = self.run_training_batch(batch, batch_nb)
85             batch_result, grad_norm_dic, batch_step_metrics = output
86 
87             # when returning -1 from train_step, we end epoch early
88             early_stop_epoch = batch_result == -1
89 
90             # ---------------
91             # RUN VAL STEP
92             # ---------------
93             is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0
94             can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
95             should_check_val = ((is_val_check_batch or early_stop_epoch) and can_check_epoch)
96 
97             # fast_dev_run always forces val checking after train batch
98             if self.fast_dev_run or should_check_val:
99                 self.run_evaluation(test=self.testing)
100 
101             # when logs should be saved
102             should_save_log = (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch
103             if should_save_log or self.fast_dev_run:
104                 if self.proc_rank == 0 and self.logger is not None:
105                     self.logger.save()
106 
107             # when metrics should be logged
108             should_log_metrics = batch_nb % self.row_log_interval == 0 or early_stop_epoch
109             if should_log_metrics or self.fast_dev_run:
110                 # logs user requested information to logger
111                 self.log_metrics(batch_step_metrics, grad_norm_dic)
112 
113             self.global_step += 1
114             self.total_batch_nb += 1
115 
116             # end epoch early
117             # stop when the flag is changed or we've gone past the amount
118             # requested in the batches
119             if early_stop_epoch or self.fast_dev_run:
120                 break
121 
122             # stop epoch if we limited nb batches
123             met_batch_limit = batch_nb >= self.nb_training_batches
124             if met_batch_limit:
125                 break
126 
127         # epoch end hook
128         if self.is_function_implemented('on_epoch_end'):
129             model = self.get_model()
130             model.on_epoch_end()
131 
132     def run_training_batch(self, batch, batch_nb):
133         # track grad norms
134         grad_norm_dic = {}
135 
136         # track all metrics for callbacks
137         all_callback_metrics = []
138 
139         # track metrics to log
140         all_log_metrics = []
141 
142         if batch is None:
143             return 0, grad_norm_dic
144 
145         # hook
146         if self.is_function_implemented('on_batch_start'):
147             model_ref = self.get_model()
148             response = model_ref.on_batch_start(batch)
149 
150             if response == -1:
151                 return -1, grad_norm_dic
152 
153         if self.show_progress_bar:
154             self.progress_bar.update(1)
155 
156         # call training_step once per optimizer
157         for opt_idx, optimizer in enumerate(self.optimizers):
158 
159             # wrap the forward step in a closure so second order methods work
160             def optimizer_closure():
161                 # forward pass
162                 output = self.training_forward(batch, batch_nb, opt_idx)
163                 closure_loss, progress_bar_metrics, log_metrics, callback_metrics = output
164 
165                 # track metrics for callbacks
166                 all_callback_metrics.append(callback_metrics)
167 
168                 # track progress bar metrics
169                 self.add_tqdm_metrics(progress_bar_metrics)
170                 all_log_metrics.append(log_metrics)
171 
172                 # accumulate loss
173                 # (if accumulate_grad_batches = 1 no effect)
174                 closure_loss = closure_loss / self.accumulate_grad_batches
175 
176                 # backward pass
177                 # done in hook so user can overwrite if needed
178                 model_ref = self.get_model()
179                 model_ref.backward(self.use_amp, closure_loss, optimizer)
180 
181                 # insert after step hook
182                 if self.is_function_implemented('on_after_backward'):
183                     model_ref = self.get_model()
184                     model_ref.on_after_backward()
185 
186                 return closure_loss
187 
188             # calculate loss
189             loss = optimizer_closure()
190 
191             # nan grads
192             if self.print_nan_grads:
193                 self.print_nan_gradients()
194 
195             # track total loss for logging (avoid mem leaks)
196             self.batch_loss_value += loss.item()
197 
198             # gradient update with accumulated gradients
199             if (self.batch_nb + 1) % self.accumulate_grad_batches == 0:
200 
201                 # track gradient norms when requested
202                 if batch_nb % self.row_log_interval == 0:
203                     if self.track_grad_norm > 0:
204                         model = self.get_model()
205                         grad_norm_dic = model.grad_norm(self.track_grad_norm)
206 
207                 # clip gradients
208                 self.clip_gradients()
209 
210                 # calls .step(), .zero_grad()
211                 # override function to modify this behavior
212                 model = self.get_model()
213                 model.optimizer_step(self.current_epoch, batch_nb,
214                                      optimizer, opt_idx, optimizer_closure)
215 
216                 # calculate running loss for display
217                 self.running_loss.append(self.batch_loss_value)
218                 self.batch_loss_value = 0
219                 self.avg_loss = np.mean(self.running_loss[-100:])
220 
221                 # update progress bar
222                 if self.show_progress_bar:
223                     # add model specific metrics
224                     tqdm_metrics = self.training_tqdm_dict
225                     self.progress_bar.set_postfix(**tqdm_metrics)
226 
227         # activate batch end hook
228         if self.is_function_implemented('on_batch_end'):
229             model = self.get_model()
230             model.on_batch_end()
231 
232         # collapse all metrics into one dict
233         all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()}
234 
235         # track all metrics for callbacks
236         self.callback_metrics = {k: v for d in all_callback_metrics for k, v in d.items()}
237 
238         return 0, grad_norm_dic, all_log_metrics
239 
240     def training_forward(self, batch, batch_nb, opt_idx):
241         """
242         Handle forward for each training case (distributed, single gpu, etc...)
243         :param batch:
244         :param batch_nb:
245         :return:
246         """
247         # ---------------
248         # FORWARD
249         # ---------------
250         # enable not needing to add opt_idx to training_step
251         args = [batch, batch_nb]
252         if len(self.optimizers) > 1:
253             args.append(opt_idx)
254 
255         if self.use_ddp or self.use_ddp2:
256             output = self.model(*args)
257         elif self.use_dp:
258             output = self.model(*args)
259         elif self.single_gpu:
260             gpu_id = 0
261             if type(self.data_parallel_device_ids) is list:
262                 gpu_id = self.data_parallel_device_ids[0]
263             batch = self.transfer_batch_to_gpu(batch, gpu_id)
264             args[0] = batch
265             output = self.model.training_step(*args)
266 
267         else:
268             output = self.model.training_step(*args)
269 
270         # format and reduce outputs accordingly
271         output = self.process_output(output, train=True)
272         loss, progress_bar_metrics, log_metrics, callback_metrics = output
273         return loss, progress_bar_metrics, log_metrics, callback_metrics
274
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py
--- a/pytorch_lightning/trainer/train_loop_mixin.py
+++ b/pytorch_lightning/trainer/train_loop_mixin.py
@@ -23,7 +23,15 @@
             # update training progress in trainer and model
             model.current_epoch = epoch_nb
             self.current_epoch = epoch_nb
-            self.total_batches = self.nb_training_batches + self.nb_val_batches
+
+            # val can be checked multiple times in epoch
+            is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
+            val_checks_per_epoch = self.nb_training_batches // self.val_check_batch
+            val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0
+
+            # total batches includes multiple val checks
+            self.total_batches = (self.nb_training_batches +
+                                  self.nb_val_batches * val_checks_per_epoch)
             self.batch_loss_value = 0  # accumulated grads
 
             # limit the number of batches to 1 in fast_dev_run

{"golden_diff": "diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py\n--- a/pytorch_lightning/trainer/train_loop_mixin.py\n+++ b/pytorch_lightning/trainer/train_loop_mixin.py\n@@ -23,7 +23,15 @@\n             # update training progress in trainer and model\n             model.current_epoch = epoch_nb\n             self.current_epoch = epoch_nb\n-            self.total_batches = self.nb_training_batches + self.nb_val_batches\n+\n+            # val can be checked multiple times in epoch\n+            is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0\n+            val_checks_per_epoch = self.nb_training_batches // self.val_check_batch\n+            val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0\n+\n+            # total batches includes multiple val checks\n+            self.total_batches = (self.nb_training_batches +\n+                                  self.nb_val_batches * val_checks_per_epoch)\n             self.batch_loss_value = 0  # accumulated grads\n \n             # limit the number of batches to 1 in fast_dev_run\n", "issue": "Incorrect total number of batches\n**Describe the bug**\r\nSometimes total number of batches is computed wrong.\r\n\r\n**To Reproduce**\r\nRun the following code with current master branch:\r\n```\r\nfrom time import sleep\r\nimport torch\r\nfrom torch.utils.data import DataLoader, Dataset\r\n\r\nimport pytorch_lightning as pl\r\n\r\n\r\nclass DummyDataset(Dataset):\r\n    def __init__(self, n):\r\n        super().__init__()\r\n        self.n = n\r\n\r\n    def __len__(self):\r\n        return self.n\r\n\r\n    def __getitem__(self, idx):\r\n        return torch.rand(10)\r\n\r\n\r\nclass CoolSystem(pl.LightningModule):\r\n    def __init__(self):\r\n        super(CoolSystem, self).__init__()\r\n        self.layer = torch.nn.Linear(10, 10)\r\n\r\n    def forward(self, x):\r\n        return self.layer(x)\r\n\r\n    def training_step(self, batch, batch_nb):\r\n        sleep(1)\r\n        return {'loss': torch.mean(self.forward(batch) ** 2)}\r\n\r\n    def validation_step(self, batch, batch_nb):\r\n        sleep(1)\r\n        return {}\r\n\r\n    def validation_end(self, outputs):\r\n        return {}\r\n\r\n    def configure_optimizers(self):\r\n        return [torch.optim.Adam(self.layer.parameters())]\r\n\r\n    @pl.data_loader\r\n    def train_dataloader(self):\r\n        return DataLoader(DummyDataset(10), batch_size=1)\r\n\r\n    @pl.data_loader\r\n    def val_dataloader(self):\r\n        return DataLoader(DummyDataset(5), batch_size=1)\r\n\r\nmodel = CoolSystem()\r\ntrainer = pl.Trainer(weights_summary=None, nb_sanity_val_steps=0, early_stop_callback=False,\r\n                     val_percent_check=1.0, val_check_interval=0.5)\r\ntrainer.fit(model)\r\n```\r\nAt first output will look like:\r\n`67%|\u2588\u2588\u2588\u2588\u2588\u258b                    | 10/15 [00:10<00:05,  1.05s/it, batch_nb=4, epoch=0, loss=0.194, v_nb=0]`\r\n\r\nBut at the end of the epoch it will be like:\r\n`20it [00:20,  1.07s/it, batch_nb=9, epoch=0, loss=0.212, v_nb=0]`\r\n\r\nMoreover, if you run\r\n```\r\ntrainer = pl.Trainer(weights_summary=None, nb_sanity_val_steps=0, early_stop_callback=False,\r\n                     val_percent_check=1.0, val_check_interval=0.5, check_val_every_n_epoch=10)\r\n```\r\nThe first epoch will end at the point \r\n`67%|\u2588\u2588\u2588\u2588\u2588\u258b                    | 10/15 [00:09<00:04,  1.01it/s, batch_nb=8, epoch=1, loss=0.069, v_nb=0]` \r\n\r\n**Expected behavior**\r\nCorrect total number of batches.\r\n\r\n**Possible solution**\r\nNow we have `total_batches = nb_training_batches + nb_val_batches`, where `nb_val_batches` is the number of batches of only one validation loop. And the problem arises because actually there can be several validation loops during one training epoch. Moreover there is a parameter `check_val_every_n_epoch` and thus there can be no validation loops at all.\r\n\r\nWith this in mind, it looks like the correct formula is:\r\n```\r\nis_val_epoch = (current_epoch + 1) % check_val_every_n_epoch == 0\r\nval_checks_per_epoch = nb_training_batches // val_check_batch if is_val_epoch else 0\r\ntotal_batches = nb_training_batches + nb_val_batches * val_checks_per_epoch\r\n```\r\n\n", "before_files": [{"content": "import numpy as np\n\ntry:\n    from apex import amp\n\n    APEX_AVAILABLE = True\nexcept ImportError:\n    APEX_AVAILABLE = False\n\n\nclass TrainerTrainLoopMixin(object):\n\n    def train(self):\n        # run all epochs\n        for epoch_nb in range(self.current_epoch, self.max_nb_epochs):\n            # set seed for distributed sampler (enables shuffling for each epoch)\n            if self.use_ddp and hasattr(self.get_train_dataloader().sampler, 'set_epoch'):\n                self.get_train_dataloader().sampler.set_epoch(epoch_nb)\n\n            # get model\n            model = self.get_model()\n\n            # update training progress in trainer and model\n            model.current_epoch = epoch_nb\n            self.current_epoch = epoch_nb\n            self.total_batches = self.nb_training_batches + self.nb_val_batches\n            self.batch_loss_value = 0  # accumulated grads\n\n            # limit the number of batches to 1 in fast_dev_run\n            if self.fast_dev_run:\n                self.total_batches = 1\n\n            # init progress_bar when requested\n            if self.show_progress_bar:\n                nb_iterations = self.total_batches\n\n                #  for iterable train loader, the progress bar never ends\n                if self.is_iterable_train_dataloader:\n                    nb_iterations = float('inf')\n                self.progress_bar.reset(nb_iterations)\n\n            # changing gradient according accumulation_scheduler\n            self.accumulation_scheduler.on_epoch_begin(epoch_nb, self)\n\n            # -----------------\n            # RUN TNG EPOCH\n            # -----------------\n            self.run_training_epoch()\n\n            # update LR schedulers\n            if self.lr_schedulers is not None:\n                for lr_scheduler in self.lr_schedulers:\n                    lr_scheduler.step(self.current_epoch)\n\n            # early stopping\n            met_min_epochs = epoch_nb > self.min_nb_epochs\n            if self.enable_early_stop and (met_min_epochs or self.fast_dev_run):\n                should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb,\n                                                                    logs=self.callback_metrics)\n                # stop training\n                stop = should_stop and met_min_epochs\n                if stop:\n                    return\n\n        if self.logger is not None:\n            self.logger.finalize(\"success\")\n\n    def run_training_epoch(self):\n        # before epoch hook\n        if self.is_function_implemented('on_epoch_start'):\n            model = self.get_model()\n            model.on_epoch_start()\n\n        # run epoch\n        for batch_nb, batch in enumerate(self.get_train_dataloader()):\n            self.batch_nb = batch_nb\n\n            model = self.get_model()\n            model.global_step = self.global_step\n\n            # ---------------\n            # RUN TRAIN STEP\n            # ---------------\n            output = self.run_training_batch(batch, batch_nb)\n            batch_result, grad_norm_dic, batch_step_metrics = output\n\n            # when returning -1 from train_step, we end epoch early\n            early_stop_epoch = batch_result == -1\n\n            # ---------------\n            # RUN VAL STEP\n            # ---------------\n            is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0\n            can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0\n            should_check_val = ((is_val_check_batch or early_stop_epoch) and can_check_epoch)\n\n            # fast_dev_run always forces val checking after train batch\n            if self.fast_dev_run or should_check_val:\n                self.run_evaluation(test=self.testing)\n\n            # when logs should be saved\n            should_save_log = (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch\n            if should_save_log or self.fast_dev_run:\n                if self.proc_rank == 0 and self.logger is not None:\n                    self.logger.save()\n\n            # when metrics should be logged\n            should_log_metrics = batch_nb % self.row_log_interval == 0 or early_stop_epoch\n            if should_log_metrics or self.fast_dev_run:\n                # logs user requested information to logger\n                self.log_metrics(batch_step_metrics, grad_norm_dic)\n\n            self.global_step += 1\n            self.total_batch_nb += 1\n\n            # end epoch early\n            # stop when the flag is changed or we've gone past the amount\n            # requested in the batches\n            if early_stop_epoch or self.fast_dev_run:\n                break\n\n            # stop epoch if we limited nb batches\n            met_batch_limit = batch_nb >= self.nb_training_batches\n            if met_batch_limit:\n                break\n\n        # epoch end hook\n        if self.is_function_implemented('on_epoch_end'):\n            model = self.get_model()\n            model.on_epoch_end()\n\n    def run_training_batch(self, batch, batch_nb):\n        # track grad norms\n        grad_norm_dic = {}\n\n        # track all metrics for callbacks\n        all_callback_metrics = []\n\n        # track metrics to log\n        all_log_metrics = []\n\n        if batch is None:\n            return 0, grad_norm_dic\n\n        # hook\n        if self.is_function_implemented('on_batch_start'):\n            model_ref = self.get_model()\n            response = model_ref.on_batch_start(batch)\n\n            if response == -1:\n                return -1, grad_norm_dic\n\n        if self.show_progress_bar:\n            self.progress_bar.update(1)\n\n        # call training_step once per optimizer\n        for opt_idx, optimizer in enumerate(self.optimizers):\n\n            # wrap the forward step in a closure so second order methods work\n            def optimizer_closure():\n                # forward pass\n                output = self.training_forward(batch, batch_nb, opt_idx)\n                closure_loss, progress_bar_metrics, log_metrics, callback_metrics = output\n\n                # track metrics for callbacks\n                all_callback_metrics.append(callback_metrics)\n\n                # track progress bar metrics\n                self.add_tqdm_metrics(progress_bar_metrics)\n                all_log_metrics.append(log_metrics)\n\n                # accumulate loss\n                # (if accumulate_grad_batches = 1 no effect)\n                closure_loss = closure_loss / self.accumulate_grad_batches\n\n                # backward pass\n                # done in hook so user can overwrite if needed\n                model_ref = self.get_model()\n                model_ref.backward(self.use_amp, closure_loss, optimizer)\n\n                # insert after step hook\n                if self.is_function_implemented('on_after_backward'):\n                    model_ref = self.get_model()\n                    model_ref.on_after_backward()\n\n                return closure_loss\n\n            # calculate loss\n            loss = optimizer_closure()\n\n            # nan grads\n            if self.print_nan_grads:\n                self.print_nan_gradients()\n\n            # track total loss for logging (avoid mem leaks)\n            self.batch_loss_value += loss.item()\n\n            # gradient update with accumulated gradients\n            if (self.batch_nb + 1) % self.accumulate_grad_batches == 0:\n\n                # track gradient norms when requested\n                if batch_nb % self.row_log_interval == 0:\n                    if self.track_grad_norm > 0:\n                        model = self.get_model()\n                        grad_norm_dic = model.grad_norm(self.track_grad_norm)\n\n                # clip gradients\n                self.clip_gradients()\n\n                # calls .step(), .zero_grad()\n                # override function to modify this behavior\n                model = self.get_model()\n                model.optimizer_step(self.current_epoch, batch_nb,\n                                     optimizer, opt_idx, optimizer_closure)\n\n                # calculate running loss for display\n                self.running_loss.append(self.batch_loss_value)\n                self.batch_loss_value = 0\n                self.avg_loss = np.mean(self.running_loss[-100:])\n\n                # update progress bar\n                if self.show_progress_bar:\n                    # add model specific metrics\n                    tqdm_metrics = self.training_tqdm_dict\n                    self.progress_bar.set_postfix(**tqdm_metrics)\n\n        # activate batch end hook\n        if self.is_function_implemented('on_batch_end'):\n            model = self.get_model()\n            model.on_batch_end()\n\n        # collapse all metrics into one dict\n        all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()}\n\n        # track all metrics for callbacks\n        self.callback_metrics = {k: v for d in all_callback_metrics for k, v in d.items()}\n\n        return 0, grad_norm_dic, all_log_metrics\n\n    def training_forward(self, batch, batch_nb, opt_idx):\n        \"\"\"\n        Handle forward for each training case (distributed, single gpu, etc...)\n        :param batch:\n        :param batch_nb:\n        :return:\n        \"\"\"\n        # ---------------\n        # FORWARD\n        # ---------------\n        # enable not needing to add opt_idx to training_step\n        args = [batch, batch_nb]\n        if len(self.optimizers) > 1:\n            args.append(opt_idx)\n\n        if self.use_ddp or self.use_ddp2:\n            output = self.model(*args)\n        elif self.use_dp:\n            output = self.model(*args)\n        elif self.single_gpu:\n            gpu_id = 0\n            if type(self.data_parallel_device_ids) is list:\n                gpu_id = self.data_parallel_device_ids[0]\n            batch = self.transfer_batch_to_gpu(batch, gpu_id)\n            args[0] = batch\n            output = self.model.training_step(*args)\n\n        else:\n            output = self.model.training_step(*args)\n\n        # format and reduce outputs accordingly\n        output = self.process_output(output, train=True)\n        loss, progress_bar_metrics, log_metrics, callback_metrics = output\n        return loss, progress_bar_metrics, log_metrics, callback_metrics\n", "path": "pytorch_lightning/trainer/train_loop_mixin.py"}], "after_files": [{"content": "import numpy as np\n\ntry:\n    from apex import amp\n\n    APEX_AVAILABLE = True\nexcept ImportError:\n    APEX_AVAILABLE = False\n\n\nclass TrainerTrainLoopMixin(object):\n\n    def train(self):\n        # run all epochs\n        for epoch_nb in range(self.current_epoch, self.max_nb_epochs):\n            # set seed for distributed sampler (enables shuffling for each epoch)\n            if self.use_ddp and hasattr(self.get_train_dataloader().sampler, 'set_epoch'):\n                self.get_train_dataloader().sampler.set_epoch(epoch_nb)\n\n            # get model\n            model = self.get_model()\n\n            # update training progress in trainer and model\n            model.current_epoch = epoch_nb\n            self.current_epoch = epoch_nb\n\n            # val can be checked multiple times in epoch\n            is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0\n            val_checks_per_epoch = self.nb_training_batches // self.val_check_batch\n            val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0\n\n            # total batches includes multiple val checks\n            self.total_batches = (self.nb_training_batches +\n                                  self.nb_val_batches * val_checks_per_epoch)\n            self.batch_loss_value = 0  # accumulated grads\n\n            # limit the number of batches to 1 in fast_dev_run\n            if self.fast_dev_run:\n                self.total_batches = 1\n\n            # init progress_bar when requested\n            if self.show_progress_bar:\n                nb_iterations = self.total_batches\n\n                #  for iterable train loader, the progress bar never ends\n                if self.is_iterable_train_dataloader:\n                    nb_iterations = float('inf')\n                self.progress_bar.reset(nb_iterations)\n\n            # changing gradient according accumulation_scheduler\n            self.accumulation_scheduler.on_epoch_begin(epoch_nb, self)\n\n            # -----------------\n            # RUN TNG EPOCH\n            # -----------------\n            self.run_training_epoch()\n\n            # update LR schedulers\n            if self.lr_schedulers is not None:\n                for lr_scheduler in self.lr_schedulers:\n                    lr_scheduler.step(self.current_epoch)\n\n            # early stopping\n            met_min_epochs = epoch_nb > self.min_nb_epochs\n            if self.enable_early_stop and (met_min_epochs or self.fast_dev_run):\n                should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb,\n                                                                    logs=self.callback_metrics)\n                # stop training\n                stop = should_stop and met_min_epochs\n                if stop:\n                    return\n\n        if self.logger is not None:\n            self.logger.finalize(\"success\")\n\n    def run_training_epoch(self):\n        # before epoch hook\n        if self.is_function_implemented('on_epoch_start'):\n            model = self.get_model()\n            model.on_epoch_start()\n\n        # run epoch\n        for batch_nb, batch in enumerate(self.get_train_dataloader()):\n            self.batch_nb = batch_nb\n\n            model = self.get_model()\n            model.global_step = self.global_step\n\n            # ---------------\n            # RUN TRAIN STEP\n            # ---------------\n            output = self.run_training_batch(batch, batch_nb)\n            batch_result, grad_norm_dic, batch_step_metrics = output\n\n            # when returning -1 from train_step, we end epoch early\n            early_stop_epoch = batch_result == -1\n\n            # ---------------\n            # RUN VAL STEP\n            # ---------------\n            is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0\n            can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0\n            should_check_val = ((is_val_check_batch or early_stop_epoch) and can_check_epoch)\n\n            # fast_dev_run always forces val checking after train batch\n            if self.fast_dev_run or should_check_val:\n                self.run_evaluation(test=self.testing)\n\n            # when logs should be saved\n            should_save_log = (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch\n            if should_save_log or self.fast_dev_run:\n                if self.proc_rank == 0 and self.logger is not None:\n                    self.logger.save()\n\n            # when metrics should be logged\n            should_log_metrics = batch_nb % self.row_log_interval == 0 or early_stop_epoch\n            if should_log_metrics or self.fast_dev_run:\n                # logs user requested information to logger\n                self.log_metrics(batch_step_metrics, grad_norm_dic)\n\n            self.global_step += 1\n            self.total_batch_nb += 1\n\n            # end epoch early\n            # stop when the flag is changed or we've gone past the amount\n            # requested in the batches\n            if early_stop_epoch or self.fast_dev_run:\n                break\n\n            # stop epoch if we limited nb batches\n            met_batch_limit = batch_nb >= self.nb_training_batches\n            if met_batch_limit:\n                break\n\n        # epoch end hook\n        if self.is_function_implemented('on_epoch_end'):\n            model = self.get_model()\n            model.on_epoch_end()\n\n    def run_training_batch(self, batch, batch_nb):\n        # track grad norms\n        grad_norm_dic = {}\n\n        # track all metrics for callbacks\n        all_callback_metrics = []\n\n        # track metrics to log\n        all_log_metrics = []\n\n        if batch is None:\n            return 0, grad_norm_dic\n\n        # hook\n        if self.is_function_implemented('on_batch_start'):\n            model_ref = self.get_model()\n            response = model_ref.on_batch_start(batch)\n\n            if response == -1:\n                return -1, grad_norm_dic\n\n        if self.show_progress_bar:\n            self.progress_bar.update(1)\n\n        # call training_step once per optimizer\n        for opt_idx, optimizer in enumerate(self.optimizers):\n\n            # wrap the forward step in a closure so second order methods work\n            def optimizer_closure():\n                # forward pass\n                output = self.training_forward(batch, batch_nb, opt_idx)\n                closure_loss, progress_bar_metrics, log_metrics, callback_metrics = output\n\n                # track metrics for callbacks\n                all_callback_metrics.append(callback_metrics)\n\n                # track progress bar metrics\n                self.add_tqdm_metrics(progress_bar_metrics)\n                all_log_metrics.append(log_metrics)\n\n                # accumulate loss\n                # (if accumulate_grad_batches = 1 no effect)\n                closure_loss = closure_loss / self.accumulate_grad_batches\n\n                # backward pass\n                # done in hook so user can overwrite if needed\n                model_ref = self.get_model()\n                model_ref.backward(self.use_amp, closure_loss, optimizer)\n\n                # insert after step hook\n                if self.is_function_implemented('on_after_backward'):\n                    model_ref = self.get_model()\n                    model_ref.on_after_backward()\n\n                return closure_loss\n\n            # calculate loss\n            loss = optimizer_closure()\n\n            # nan grads\n            if self.print_nan_grads:\n                self.print_nan_gradients()\n\n            # track total loss for logging (avoid mem leaks)\n            self.batch_loss_value += loss.item()\n\n            # gradient update with accumulated gradients\n            if (self.batch_nb + 1) % self.accumulate_grad_batches == 0:\n\n                # track gradient norms when requested\n                if batch_nb % self.row_log_interval == 0:\n                    if self.track_grad_norm > 0:\n                        model = self.get_model()\n                        grad_norm_dic = model.grad_norm(self.track_grad_norm)\n\n                # clip gradients\n                self.clip_gradients()\n\n                # calls .step(), .zero_grad()\n                # override function to modify this behavior\n                model = self.get_model()\n                model.optimizer_step(self.current_epoch, batch_nb,\n                                     optimizer, opt_idx, optimizer_closure)\n\n                # calculate running loss for display\n                self.running_loss.append(self.batch_loss_value)\n                self.batch_loss_value = 0\n                self.avg_loss = np.mean(self.running_loss[-100:])\n\n                # update progress bar\n                if self.show_progress_bar:\n                    # add model specific metrics\n                    tqdm_metrics = self.training_tqdm_dict\n                    self.progress_bar.set_postfix(**tqdm_metrics)\n\n        # activate batch end hook\n        if self.is_function_implemented('on_batch_end'):\n            model = self.get_model()\n            model.on_batch_end()\n\n        # collapse all metrics into one dict\n        all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()}\n\n        # track all metrics for callbacks\n        self.callback_metrics = {k: v for d in all_callback_metrics for k, v in d.items()}\n\n        return 0, grad_norm_dic, all_log_metrics\n\n    def training_forward(self, batch, batch_nb, opt_idx):\n        \"\"\"\n        Handle forward for each training case (distributed, single gpu, etc...)\n        :param batch:\n        :param batch_nb:\n        :return:\n        \"\"\"\n        # ---------------\n        # FORWARD\n        # ---------------\n        # enable not needing to add opt_idx to training_step\n        args = [batch, batch_nb]\n        if len(self.optimizers) > 1:\n            args.append(opt_idx)\n\n        if self.use_ddp or self.use_ddp2:\n            output = self.model(*args)\n        elif self.use_dp:\n            output = self.model(*args)\n        elif self.single_gpu:\n            gpu_id = 0\n            if type(self.data_parallel_device_ids) is list:\n                gpu_id = self.data_parallel_device_ids[0]\n            batch = self.transfer_batch_to_gpu(batch, gpu_id)\n            args[0] = batch\n            output = self.model.training_step(*args)\n\n        else:\n            output = self.model.training_step(*args)\n\n        # format and reduce outputs accordingly\n        output = self.process_output(output, train=True)\n        loss, progress_bar_metrics, log_metrics, callback_metrics = output\n        return loss, progress_bar_metrics, log_metrics, callback_metrics\n", "path": "pytorch_lightning/trainer/train_loop_mixin.py"}]}