def training_step_and_backward(self, split_batch, batch_idx, opt_idx, optimizer, hiddens):
    'Wrap forward, zero_grad and backward in a closure so second order methods work'
    with self.trainer.profiler.profile('training_step_and_backward'):
        result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
        self._curr_step_result = result
        if ((not self._skip_backward) and self.trainer.lightning_module.automatic_optimization):
            is_first_batch_to_accumulate = ((batch_idx % self.trainer.accumulate_grad_batches) == 0)
            if is_first_batch_to_accumulate:
                self.on_before_zero_grad(optimizer)
                self.optimizer_zero_grad(batch_idx, optimizer, opt_idx)
            if (result is not None):
                with self.trainer.profiler.profile('backward'):
                    self.backward(result, optimizer, opt_idx)
                if (not self.should_accumulate()):
                    self.on_after_backward(result.training_step_output, batch_idx, result.loss)
                if self.trainer.terminate_on_nan:
                    self._check_finite(result.loss)
            else:
                self.warning_cache.warn('training_step returned None. If this was on purpose, ignore this warning...')
            if (len(self.trainer.optimizers) > 1):
                self.trainer.lightning_module.untoggle_optimizer(opt_idx)
    return result
Wrap forward, zero_grad and backward in a closure so second order methods work
def training_step_and_backward(self, split_batch, batch_idx, opt_idx, optimizer, hiddens):
    
    with self.trainer.profiler.profile('training_step_and_backward'):
        result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
        self._curr_step_result = result
        if ((not self._skip_backward) and self.trainer.lightning_module.automatic_optimization):
            is_first_batch_to_accumulate = ((batch_idx % self.trainer.accumulate_grad_batches) == 0)
            if is_first_batch_to_accumulate:
                self.on_before_zero_grad(optimizer)
                self.optimizer_zero_grad(batch_idx, optimizer, opt_idx)
            if (result is not None):
                with self.trainer.profiler.profile('backward'):
                    self.backward(result, optimizer, opt_idx)
                if (not self.should_accumulate()):
                    self.on_after_backward(result.training_step_output, batch_idx, result.loss)
                if self.trainer.terminate_on_nan:
                    self._check_finite(result.loss)
            else:
                self.warning_cache.warn('training_step returned None. If this was on purpose, ignore this warning...')
            if (len(self.trainer.optimizers) > 1):
                self.trainer.lightning_module.untoggle_optimizer(opt_idx)
    return result