prompt: You will be provided with a partial code base and an issue statement explaining a problem to resolve.
  <issue>
  Performance drop when activating gradient clipping
  Hello all,
  
  I experienced a substantial drop in computation time when activating gradient clipping (by passing a non-zero value to the keyword argument `gradient_clip_val` when initializing the Trainer).
  
  I noticed that in the current implementation of the `clipping_gradient` method in pytorch-lightning/trainer/training_tricks.py redundant computations are made by first computing the 2-norm and second squaring this result, which could be shortened by computing the sum of squares directly. This saves one square root and squaring operation per parameter set.
  
  Best,
  Jonas
  
  ### Environment
  
  ```
  cuda:
  	GPU:
  	available:           False
  	version:             None
  packages:
  	numpy:               1.18.1
  	pyTorch_debug:       False
  	pyTorch_version:     1.4.0
  	pytorch-lightning:   0.7.4-dev
  	tensorboard:         2.2.1
  	tqdm:                4.45.0
  system:
  	OS:                  Darwin
  	architecture:
  		64bit
  		
  	processor:           i386
  	python:              3.8.2
  	version:             Darwin Kernel Version 19.4.0: Wed Mar  4 22:28:40 PST 2020; root:xnu-6153.101.6~15/RELEASE_X86_64
  ```
  
  ### Additional context
  
  I trained a relatively small (two-layered) MLP on MNIST; perhaps this performance drop does not become that apparent when training on larger network architectures.
  
  </issue>
  <code>
  [start of pytorch_lightning/trainer/training_tricks.py]
  1 import math
  2 import sys
  3 from abc import ABC, abstractmethod
  4 
  5 import torch
  6 from torch import Tensor
  7 
  8 from pytorch_lightning import _logger as log
  9 from pytorch_lightning.callbacks import GradientAccumulationScheduler
  10 
  11 EPSILON = 1e-6
  12 EPSILON_FP16 = 1e-5
  13 
  14 
  15 class TrainerTrainingTricksMixin(ABC):
  16 
  17     # this is just a summary on variables used in this abstract class,
  18     #  the proper values/initialisation should be done in child class
  19     gradient_clip_val: ...
  20     precision: ...
  21 
  22     @abstractmethod
  23     def get_model(self):
  24         """Warning: this is just empty shell for code implemented in other class."""
  25 
  26     def clip_gradients(self):
  27         # this code is a modification of torch.nn.utils.clip_grad_norm_
  28         # with TPU support based on https://github.com/pytorch/xla/blob/master/TROUBLESHOOTING.md
  29         if self.gradient_clip_val > 0:
  30             model = self.get_model()
  31             parameters = model.parameters()
  32             max_norm = float(self.gradient_clip_val)
  33             norm_type = float(2.0)
  34             if isinstance(parameters, torch.Tensor):
  35                 parameters = [parameters]
  36             parameters = list(filter(lambda p: p.grad is not None, parameters))
  37             if norm_type == math.inf:
  38                 total_norm = max(p.grad.data.abs().max() for p in parameters)
  39             else:
  40                 device = parameters[0].device
  41                 total_norm = torch.zeros([], device=device if parameters else None)
  42                 for p in parameters:
  43                     param_norm = p.grad.data.norm(norm_type) ** norm_type
  44                     total_norm.add_(param_norm)
  45                 total_norm = (total_norm ** (1. / norm_type))
  46             eps = EPSILON_FP16 if self.precision == 16 else EPSILON
  47             clip_coef = torch.tensor(max_norm, device=device) / (total_norm + eps)
  48             for p in parameters:
  49                 p.grad.data.mul_(torch.where(clip_coef < 1, clip_coef, torch.tensor(1., device=device)))
  50 
  51     def print_nan_gradients(self) -> None:
  52         model = self.get_model()
  53         for param in model.parameters():
  54             if (param.grad is not None) and torch.isnan(param.grad.float()).any():
  55                 log.info(param, param.grad)
  56 
  57     def detect_nan_tensors(self, loss: Tensor) -> None:
  58         model = self.get_model()
  59 
  60         # check if loss is nan
  61         if not torch.isfinite(loss).all():
  62             raise ValueError(
  63                 'The loss returned in `training_step` is nan or inf.'
  64             )
  65         # check if a network weight is nan
  66         for name, param in model.named_parameters():
  67             if not torch.isfinite(param).all():
  68                 self.print_nan_gradients()
  69                 raise ValueError(
  70                     f'Detected nan and/or inf values in `{name}`.'
  71                     ' Check your forward pass for numerically unstable operations.'
  72                 )
  73 
  74     def configure_accumulated_gradients(self, accumulate_grad_batches):
  75         if isinstance(accumulate_grad_batches, dict):
  76             self.accumulation_scheduler = GradientAccumulationScheduler(accumulate_grad_batches)
  77         elif isinstance(accumulate_grad_batches, int):
  78             schedule = {1: accumulate_grad_batches}
  79             self.accumulation_scheduler = GradientAccumulationScheduler(schedule)
  80         else:
  81             raise TypeError("Gradient accumulation supports only int and dict types")
  82
  [end of pytorch_lightning/trainer/training_tricks.py]
  </code>
  
  I need you to solve this issue by generating a single patch file that I can apply directly to this repository using git apply. Please respond with a single patch file in the following format.
  <patch>
  diff --git a/file.py b/file.py
  --- a/file.py
  +++ b/file.py
  @@ -1,27 +1,35 @@
   def euclidean(a, b):
  -    while b:
  -        a, b = b, a % b
  -    return a
  +    if b == 0:
  +        return a
  +    return euclidean(b, a % b)
   
   
   def bresenham(x0, y0, x1, y1):
       points = []
       dx = abs(x1 - x0)
       dy = abs(y1 - y0)
  -    sx = 1 if x0 < x1 else -1
  -    sy = 1 if y0 < y1 else -1
  -    err = dx - dy
  +    x, y = x0, y0
  +    sx = -1 if x0 > x1 else 1
  +    sy = -1 if y0 > y1 else 1
   
  -    while True:
  -        points.append((x0, y0))
  -        if x0 == x1 and y0 == y1:
  -            break
  -        e2 = 2 * err
  -        if e2 > -dy:
  -            err -= dy
  -            x0 += sx
  -        if e2 < dx:
  -            err += dx
  -            y0 += sy
  +    if dx > dy:
  +        err = dx / 2.0
  +        while x != x1:
  +            points.append((x, y))
  +            err -= dy
  +            if err < 0:
  +                y += sy
  +                err += dx
  +            x += sx
  +    else:
  +        err = dy / 2.0
  +        while y != y1:
  +            points.append((x, y))
  +            err -= dx
  +            if err < 0:
  +                x += sx
  +                err += dy
  +            y += sy
  +
  +    points.append((x, y))
       return points
  
  </patch>
golden_diff: diff --git a/pytorch_lightning/trainer/training_tricks.py b/pytorch_lightning/trainer/training_tricks.py
  --- a/pytorch_lightning/trainer/training_tricks.py
  +++ b/pytorch_lightning/trainer/training_tricks.py
  @@ -40,7 +40,7 @@
                   device = parameters[0].device
                   total_norm = torch.zeros([], device=device if parameters else None)
                   for p in parameters:
  -                    param_norm = p.grad.data.norm(norm_type) ** norm_type
  +                    param_norm = p.grad.data.pow(norm_type).sum()
                       total_norm.add_(param_norm)
                   total_norm = (total_norm ** (1. / norm_type))
               eps = EPSILON_FP16 if self.precision == 16 else EPSILON
  
verification_info: {"golden_diff": "diff --git a/pytorch_lightning/trainer/training_tricks.py b/pytorch_lightning/trainer/training_tricks.py\n--- a/pytorch_lightning/trainer/training_tricks.py\n+++ b/pytorch_lightning/trainer/training_tricks.py\n@@ -40,7 +40,7 @@\n                 device = parameters[0].device\n                 total_norm = torch.zeros([], device=device if parameters else None)\n                 for p in parameters:\n-                    param_norm = p.grad.data.norm(norm_type) ** norm_type\n+                    param_norm = p.grad.data.pow(norm_type).sum()\n                     total_norm.add_(param_norm)\n                 total_norm = (total_norm ** (1. / norm_type))\n             eps = EPSILON_FP16 if self.precision == 16 else EPSILON\n", "issue": "Performance drop when activating gradient clipping\nHello all,\r\n\r\nI experienced a substantial drop in computation time when activating gradient clipping (by passing a non-zero value to the keyword argument `gradient_clip_val` when initializing the Trainer).\r\n\r\nI noticed that in the current implementation of the `clipping_gradient` method in pytorch-lightning/trainer/training_tricks.py redundant computations are made by first computing the 2-norm and second squaring this result, which could be shortened by computing the sum of squares directly. This saves one square root and squaring operation per parameter set.\r\n\r\nBest,\r\nJonas\r\n\r\n### Environment\r\n\r\n```\r\ncuda:\r\n\tGPU:\r\n\tavailable:           False\r\n\tversion:             None\r\npackages:\r\n\tnumpy:               1.18.1\r\n\tpyTorch_debug:       False\r\n\tpyTorch_version:     1.4.0\r\n\tpytorch-lightning:   0.7.4-dev\r\n\ttensorboard:         2.2.1\r\n\ttqdm:                4.45.0\r\nsystem:\r\n\tOS:                  Darwin\r\n\tarchitecture:\r\n\t\t64bit\r\n\t\t\r\n\tprocessor:           i386\r\n\tpython:              3.8.2\r\n\tversion:             Darwin Kernel Version 19.4.0: Wed Mar  4 22:28:40 PST 2020; root:xnu-6153.101.6~15/RELEASE_X86_64\r\n```\r\n\r\n### Additional context\r\n\r\nI trained a relatively small (two-layered) MLP on MNIST; perhaps this performance drop does not become that apparent when training on larger network architectures.\n", "before_files": [{"content": "import math\nimport sys\nfrom abc import ABC, abstractmethod\n\nimport torch\nfrom torch import Tensor\n\nfrom pytorch_lightning import _logger as log\nfrom pytorch_lightning.callbacks import GradientAccumulationScheduler\n\nEPSILON = 1e-6\nEPSILON_FP16 = 1e-5\n\n\nclass TrainerTrainingTricksMixin(ABC):\n\n    # this is just a summary on variables used in this abstract class,\n    #  the proper values/initialisation should be done in child class\n    gradient_clip_val: ...\n    precision: ...\n\n    @abstractmethod\n    def get_model(self):\n        \"\"\"Warning: this is just empty shell for code implemented in other class.\"\"\"\n\n    def clip_gradients(self):\n        # this code is a modification of torch.nn.utils.clip_grad_norm_\n        # with TPU support based on https://github.com/pytorch/xla/blob/master/TROUBLESHOOTING.md\n        if self.gradient_clip_val > 0:\n            model = self.get_model()\n            parameters = model.parameters()\n            max_norm = float(self.gradient_clip_val)\n            norm_type = float(2.0)\n            if isinstance(parameters, torch.Tensor):\n                parameters = [parameters]\n            parameters = list(filter(lambda p: p.grad is not None, parameters))\n            if norm_type == math.inf:\n                total_norm = max(p.grad.data.abs().max() for p in parameters)\n            else:\n                device = parameters[0].device\n                total_norm = torch.zeros([], device=device if parameters else None)\n                for p in parameters:\n                    param_norm = p.grad.data.norm(norm_type) ** norm_type\n                    total_norm.add_(param_norm)\n                total_norm = (total_norm ** (1. / norm_type))\n            eps = EPSILON_FP16 if self.precision == 16 else EPSILON\n            clip_coef = torch.tensor(max_norm, device=device) / (total_norm + eps)\n            for p in parameters:\n                p.grad.data.mul_(torch.where(clip_coef < 1, clip_coef, torch.tensor(1., device=device)))\n\n    def print_nan_gradients(self) -> None:\n        model = self.get_model()\n        for param in model.parameters():\n            if (param.grad is not None) and torch.isnan(param.grad.float()).any():\n                log.info(param, param.grad)\n\n    def detect_nan_tensors(self, loss: Tensor) -> None:\n        model = self.get_model()\n\n        # check if loss is nan\n        if not torch.isfinite(loss).all():\n            raise ValueError(\n                'The loss returned in `training_step` is nan or inf.'\n            )\n        # check if a network weight is nan\n        for name, param in model.named_parameters():\n            if not torch.isfinite(param).all():\n                self.print_nan_gradients()\n                raise ValueError(\n                    f'Detected nan and/or inf values in `{name}`.'\n                    ' Check your forward pass for numerically unstable operations.'\n                )\n\n    def configure_accumulated_gradients(self, accumulate_grad_batches):\n        if isinstance(accumulate_grad_batches, dict):\n            self.accumulation_scheduler = GradientAccumulationScheduler(accumulate_grad_batches)\n        elif isinstance(accumulate_grad_batches, int):\n            schedule = {1: accumulate_grad_batches}\n            self.accumulation_scheduler = GradientAccumulationScheduler(schedule)\n        else:\n            raise TypeError(\"Gradient accumulation supports only int and dict types\")\n", "path": "pytorch_lightning/trainer/training_tricks.py"}]}