prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  speed up NanGuardMode on GPU and move to Theano.
  This can be done as in gh-1054. Do the reduction on the GPU, then this will transfer much less data.
  
  The CudaNdarray object do not support many reduction, but we can compile a Theano function that take a gpu object, do the reduction and return the result on the CPU to inspect it.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `pylearn2/devtools/nan_guard.py`
  Content:
  ```
  1 """
  2 Functionality for detecting NaNs in a Theano graph.
  3 """
  4 __authors__ = "Ian Goodfellow"
  5 __copyright__ = "Copyright 2010-2012, Universite de Montreal"
  6 __credits__ = ["Ian Goodfellow"]
  7 __license__ = "3-clause BSD"
  8 __maintainer__ = "LISA Lab"
  9 __email__ = "pylearn-dev@googlegroups"
  10 
  11 import logging
  12 from theano.compile import Mode
  13 import theano
  14 import numpy as np
  15 from pylearn2.models.dbm import flatten
  16 from pylearn2.utils import contains_nan, contains_inf
  17 
  18 
  19 logger = logging.getLogger(__name__)
  20 
  21 
  22 class NanGuardMode(Mode):
  23     """
  24     A Theano compilation Mode that makes the compiled function automatically
  25     detect NaNs and Infs and detect an error if they occur.
  26 
  27     Parameters
  28     ----------
  29     nan_is_error : bool
  30         If True, raise an error anytime a NaN is encountered
  31     inf_is_error: bool
  32         If True, raise an error anytime an Inf is encountered.  Note that some
  33         pylearn2 modules currently use np.inf as a default value (e.g.
  34         mlp.max_pool) and these will cause an error if inf_is_error is True.
  35     big_is_error: bool
  36         If True, raise an error when a value greater than 1e10 is encountered.
  37     """
  38     def __init__(self, nan_is_error, inf_is_error, big_is_error=True):
  39         def do_check_on(var, nd, f, is_input):
  40             """
  41             Checks `var` for NaNs / Infs. If detected, raises an exception
  42             and / or prints information about `nd`, `f`, and `is_input` to
  43             help the user determine the cause of the invalid values.
  44 
  45             Parameters
  46             ----------
  47             var : numpy.ndarray
  48                 The value to be checked.
  49             nd : theano.gof.Apply
  50                 The Apply node being executed
  51             f : callable
  52                 The thunk for the apply node
  53             is_input : bool
  54                 If True, `var` is an input to `nd`.
  55                 If False, it is an output.
  56             """
  57             error = False
  58             if nan_is_error:
  59                 if contains_nan(var):
  60                     logger.error('NaN detected')
  61                     error = True
  62             if inf_is_error:
  63                 if contains_inf(var):
  64                     logger.error('Inf detected')
  65                     error = True
  66             if big_is_error:
  67                 if np.abs(var).max() > 1e10:
  68                     logger.error('Big value detected')
  69                     error = True
  70             if error:
  71                 if is_input:
  72                     logger.error('In an input')
  73                 else:
  74                     logger.error('In an output')
  75                 logger.error('Inputs: ')
  76                 for ivar, ival in zip(nd.inputs, f.inputs):
  77                     logger.error('var')
  78                     logger.error(ivar)
  79                     logger.error(theano.printing.min_informative_str(ivar))
  80                     logger.error('val')
  81                     logger.error(ival)
  82                 logger.error('Node:')
  83                 logger.error(nd)
  84                 assert False
  85 
  86         def nan_check(i, node, fn):
  87             """
  88             Runs `fn` while checking its inputs and outputs for NaNs / Infs
  89 
  90             Parameters
  91             ----------
  92             i : currently ignored (TODO: determine why it is here or remove)
  93             node : theano.gof.Apply
  94                 The Apply node currently being executed
  95             fn : callable
  96                 The thunk to execute for this Apply node
  97             """
  98             inputs = fn.inputs
  99             # TODO: figure out why individual inputs are themselves lists sometimes
  100             for x in flatten(inputs):
  101                 do_check_on(x, node, fn, True)
  102             fn()
  103             outputs = fn.outputs
  104             for j, x in enumerate(flatten(outputs)):
  105                 do_check_on(x, node, fn, False)
  106 
  107         wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()], [nan_check])
  108         super(NanGuardMode, self).__init__(wrap_linker, optimizer=theano.config.optimizer)
  109
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/pylearn2/devtools/nan_guard.py b/pylearn2/devtools/nan_guard.py
  --- a/pylearn2/devtools/nan_guard.py
  +++ b/pylearn2/devtools/nan_guard.py
  @@ -11,6 +11,8 @@
   import logging
   from theano.compile import Mode
   import theano
  +import theano.tensor as T
  +import theano.sandbox.cuda as cuda
   import numpy as np
   from pylearn2.models.dbm import flatten
   from pylearn2.utils import contains_nan, contains_inf
  @@ -36,6 +38,23 @@
           If True, raise an error when a value greater than 1e10 is encountered.
       """
       def __init__(self, nan_is_error, inf_is_error, big_is_error=True):
  +        if cuda.cuda_available:
  +            self.guard_input = cuda.fvector('nan_guard')
  +            if nan_is_error or inf_is_error:
  +                self.gpumin = theano.function(
  +                    [self.guard_input], T.min(self.guard_input),
  +                    mode='FAST_RUN'
  +                )
  +            if inf_is_error:
  +                self.gpumax = theano.function(
  +                    [self.guard_input], T.max(self.guard_input),
  +                    mode='FAST_RUN'
  +                )
  +            if big_is_error:
  +                self.gpuabsmax = theano.function(
  +                    [self.guard_input], T.max(T.abs_(self.guard_input)),
  +                    mode='FAST_RUN'
  +                )
           def do_check_on(var, nd, f, is_input):
               """
               Checks `var` for NaNs / Infs. If detected, raises an exception
  @@ -56,15 +75,31 @@
               """
               error = False
               if nan_is_error:
  -                if contains_nan(var):
  +                err = False
  +                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
  +                    err = np.isnan(self.gpumin(var.reshape(var.size)))
  +                else:
  +                    err = contains_nan(var)
  +                if err:
                       logger.error('NaN detected')
                       error = True
               if inf_is_error:
  -                if contains_inf(var):
  +                err = False
  +                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
  +                    err = (np.isinf(self.gpumin(var.reshape(var.size))) or \
  +                           np.isinf(self.gpumax(var.reshape(var.size))))
  +                else:
  +                    err = contains_inf(var)
  +                if err:
                       logger.error('Inf detected')
                       error = True
               if big_is_error:
  -                if np.abs(var).max() > 1e10:
  +                err = False
  +                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):
  +                    err = (self.gpuabsmax(var.reshape(var.size)) > 1e10)
  +                else:
  +                    err = (np.abs(var).max() > 1e10)
  +                if err:
                       logger.error('Big value detected')
                       error = True
               if error:
  
verification_info: {"golden_diff": "diff --git a/pylearn2/devtools/nan_guard.py b/pylearn2/devtools/nan_guard.py\n--- a/pylearn2/devtools/nan_guard.py\n+++ b/pylearn2/devtools/nan_guard.py\n@@ -11,6 +11,8 @@\n import logging\n from theano.compile import Mode\n import theano\n+import theano.tensor as T\n+import theano.sandbox.cuda as cuda\n import numpy as np\n from pylearn2.models.dbm import flatten\n from pylearn2.utils import contains_nan, contains_inf\n@@ -36,6 +38,23 @@\n         If True, raise an error when a value greater than 1e10 is encountered.\n     \"\"\"\n     def __init__(self, nan_is_error, inf_is_error, big_is_error=True):\n+        if cuda.cuda_available:\n+            self.guard_input = cuda.fvector('nan_guard')\n+            if nan_is_error or inf_is_error:\n+                self.gpumin = theano.function(\n+                    [self.guard_input], T.min(self.guard_input),\n+                    mode='FAST_RUN'\n+                )\n+            if inf_is_error:\n+                self.gpumax = theano.function(\n+                    [self.guard_input], T.max(self.guard_input),\n+                    mode='FAST_RUN'\n+                )\n+            if big_is_error:\n+                self.gpuabsmax = theano.function(\n+                    [self.guard_input], T.max(T.abs_(self.guard_input)),\n+                    mode='FAST_RUN'\n+                )\n         def do_check_on(var, nd, f, is_input):\n             \"\"\"\n             Checks `var` for NaNs / Infs. If detected, raises an exception\n@@ -56,15 +75,31 @@\n             \"\"\"\n             error = False\n             if nan_is_error:\n-                if contains_nan(var):\n+                err = False\n+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n+                    err = np.isnan(self.gpumin(var.reshape(var.size)))\n+                else:\n+                    err = contains_nan(var)\n+                if err:\n                     logger.error('NaN detected')\n                     error = True\n             if inf_is_error:\n-                if contains_inf(var):\n+                err = False\n+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n+                    err = (np.isinf(self.gpumin(var.reshape(var.size))) or \\\n+                           np.isinf(self.gpumax(var.reshape(var.size))))\n+                else:\n+                    err = contains_inf(var)\n+                if err:\n                     logger.error('Inf detected')\n                     error = True\n             if big_is_error:\n-                if np.abs(var).max() > 1e10:\n+                err = False\n+                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n+                    err = (self.gpuabsmax(var.reshape(var.size)) > 1e10)\n+                else:\n+                    err = (np.abs(var).max() > 1e10)\n+                if err:\n                     logger.error('Big value detected')\n                     error = True\n             if error:\n", "issue": "speed up NanGuardMode on GPU and move to Theano.\nThis can be done as in gh-1054. Do the reduction on the GPU, then this will transfer much less data.\n\nThe CudaNdarray object do not support many reduction, but we can compile a Theano function that take a gpu object, do the reduction and return the result on the CPU to inspect it.\n\n", "before_files": [{"content": "\"\"\"\nFunctionality for detecting NaNs in a Theano graph.\n\"\"\"\n__authors__ = \"Ian Goodfellow\"\n__copyright__ = \"Copyright 2010-2012, Universite de Montreal\"\n__credits__ = [\"Ian Goodfellow\"]\n__license__ = \"3-clause BSD\"\n__maintainer__ = \"LISA Lab\"\n__email__ = \"pylearn-dev@googlegroups\"\n\nimport logging\nfrom theano.compile import Mode\nimport theano\nimport numpy as np\nfrom pylearn2.models.dbm import flatten\nfrom pylearn2.utils import contains_nan, contains_inf\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass NanGuardMode(Mode):\n    \"\"\"\n    A Theano compilation Mode that makes the compiled function automatically\n    detect NaNs and Infs and detect an error if they occur.\n\n    Parameters\n    ----------\n    nan_is_error : bool\n        If True, raise an error anytime a NaN is encountered\n    inf_is_error: bool\n        If True, raise an error anytime an Inf is encountered.  Note that some\n        pylearn2 modules currently use np.inf as a default value (e.g.\n        mlp.max_pool) and these will cause an error if inf_is_error is True.\n    big_is_error: bool\n        If True, raise an error when a value greater than 1e10 is encountered.\n    \"\"\"\n    def __init__(self, nan_is_error, inf_is_error, big_is_error=True):\n        def do_check_on(var, nd, f, is_input):\n            \"\"\"\n            Checks `var` for NaNs / Infs. If detected, raises an exception\n            and / or prints information about `nd`, `f`, and `is_input` to\n            help the user determine the cause of the invalid values.\n\n            Parameters\n            ----------\n            var : numpy.ndarray\n                The value to be checked.\n            nd : theano.gof.Apply\n                The Apply node being executed\n            f : callable\n                The thunk for the apply node\n            is_input : bool\n                If True, `var` is an input to `nd`.\n                If False, it is an output.\n            \"\"\"\n            error = False\n            if nan_is_error:\n                if contains_nan(var):\n                    logger.error('NaN detected')\n                    error = True\n            if inf_is_error:\n                if contains_inf(var):\n                    logger.error('Inf detected')\n                    error = True\n            if big_is_error:\n                if np.abs(var).max() > 1e10:\n                    logger.error('Big value detected')\n                    error = True\n            if error:\n                if is_input:\n                    logger.error('In an input')\n                else:\n                    logger.error('In an output')\n                logger.error('Inputs: ')\n                for ivar, ival in zip(nd.inputs, f.inputs):\n                    logger.error('var')\n                    logger.error(ivar)\n                    logger.error(theano.printing.min_informative_str(ivar))\n                    logger.error('val')\n                    logger.error(ival)\n                logger.error('Node:')\n                logger.error(nd)\n                assert False\n\n        def nan_check(i, node, fn):\n            \"\"\"\n            Runs `fn` while checking its inputs and outputs for NaNs / Infs\n\n            Parameters\n            ----------\n            i : currently ignored (TODO: determine why it is here or remove)\n            node : theano.gof.Apply\n                The Apply node currently being executed\n            fn : callable\n                The thunk to execute for this Apply node\n            \"\"\"\n            inputs = fn.inputs\n            # TODO: figure out why individual inputs are themselves lists sometimes\n            for x in flatten(inputs):\n                do_check_on(x, node, fn, True)\n            fn()\n            outputs = fn.outputs\n            for j, x in enumerate(flatten(outputs)):\n                do_check_on(x, node, fn, False)\n\n        wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()], [nan_check])\n        super(NanGuardMode, self).__init__(wrap_linker, optimizer=theano.config.optimizer)\n", "path": "pylearn2/devtools/nan_guard.py"}], "after_files": [{"content": "\"\"\"\nFunctionality for detecting NaNs in a Theano graph.\n\"\"\"\n__authors__ = \"Ian Goodfellow\"\n__copyright__ = \"Copyright 2010-2012, Universite de Montreal\"\n__credits__ = [\"Ian Goodfellow\"]\n__license__ = \"3-clause BSD\"\n__maintainer__ = \"LISA Lab\"\n__email__ = \"pylearn-dev@googlegroups\"\n\nimport logging\nfrom theano.compile import Mode\nimport theano\nimport theano.tensor as T\nimport theano.sandbox.cuda as cuda\nimport numpy as np\nfrom pylearn2.models.dbm import flatten\nfrom pylearn2.utils import contains_nan, contains_inf\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass NanGuardMode(Mode):\n    \"\"\"\n    A Theano compilation Mode that makes the compiled function automatically\n    detect NaNs and Infs and detect an error if they occur.\n\n    Parameters\n    ----------\n    nan_is_error : bool\n        If True, raise an error anytime a NaN is encountered\n    inf_is_error: bool\n        If True, raise an error anytime an Inf is encountered.  Note that some\n        pylearn2 modules currently use np.inf as a default value (e.g.\n        mlp.max_pool) and these will cause an error if inf_is_error is True.\n    big_is_error: bool\n        If True, raise an error when a value greater than 1e10 is encountered.\n    \"\"\"\n    def __init__(self, nan_is_error, inf_is_error, big_is_error=True):\n        if cuda.cuda_available:\n            self.guard_input = cuda.fvector('nan_guard')\n            if nan_is_error or inf_is_error:\n                self.gpumin = theano.function(\n                    [self.guard_input], T.min(self.guard_input),\n                    mode='FAST_RUN'\n                )\n            if inf_is_error:\n                self.gpumax = theano.function(\n                    [self.guard_input], T.max(self.guard_input),\n                    mode='FAST_RUN'\n                )\n            if big_is_error:\n                self.gpuabsmax = theano.function(\n                    [self.guard_input], T.max(T.abs_(self.guard_input)),\n                    mode='FAST_RUN'\n                )\n        def do_check_on(var, nd, f, is_input):\n            \"\"\"\n            Checks `var` for NaNs / Infs. If detected, raises an exception\n            and / or prints information about `nd`, `f`, and `is_input` to\n            help the user determine the cause of the invalid values.\n\n            Parameters\n            ----------\n            var : numpy.ndarray\n                The value to be checked.\n            nd : theano.gof.Apply\n                The Apply node being executed\n            f : callable\n                The thunk for the apply node\n            is_input : bool\n                If True, `var` is an input to `nd`.\n                If False, it is an output.\n            \"\"\"\n            error = False\n            if nan_is_error:\n                err = False\n                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n                    err = np.isnan(self.gpumin(var.reshape(var.size)))\n                else:\n                    err = contains_nan(var)\n                if err:\n                    logger.error('NaN detected')\n                    error = True\n            if inf_is_error:\n                err = False\n                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n                    err = (np.isinf(self.gpumin(var.reshape(var.size))) or \\\n                           np.isinf(self.gpumax(var.reshape(var.size))))\n                else:\n                    err = contains_inf(var)\n                if err:\n                    logger.error('Inf detected')\n                    error = True\n            if big_is_error:\n                err = False\n                if cuda.cuda_available and isinstance(var, cuda.CudaNdarray):\n                    err = (self.gpuabsmax(var.reshape(var.size)) > 1e10)\n                else:\n                    err = (np.abs(var).max() > 1e10)\n                if err:\n                    logger.error('Big value detected')\n                    error = True\n            if error:\n                if is_input:\n                    logger.error('In an input')\n                else:\n                    logger.error('In an output')\n                logger.error('Inputs: ')\n                for ivar, ival in zip(nd.inputs, f.inputs):\n                    logger.error('var')\n                    logger.error(ivar)\n                    logger.error(theano.printing.min_informative_str(ivar))\n                    logger.error('val')\n                    logger.error(ival)\n                logger.error('Node:')\n                logger.error(nd)\n                assert False\n\n        def nan_check(i, node, fn):\n            \"\"\"\n            Runs `fn` while checking its inputs and outputs for NaNs / Infs\n\n            Parameters\n            ----------\n            i : currently ignored (TODO: determine why it is here or remove)\n            node : theano.gof.Apply\n                The Apply node currently being executed\n            fn : callable\n                The thunk to execute for this Apply node\n            \"\"\"\n            inputs = fn.inputs\n            # TODO: figure out why individual inputs are themselves lists sometimes\n            for x in flatten(inputs):\n                do_check_on(x, node, fn, True)\n            fn()\n            outputs = fn.outputs\n            for j, x in enumerate(flatten(outputs)):\n                do_check_on(x, node, fn, False)\n\n        wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()], [nan_check])\n        super(NanGuardMode, self).__init__(wrap_linker, optimizer=theano.config.optimizer)\n", "path": "pylearn2/devtools/nan_guard.py"}]}