We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
2nd order derivative of ELU should not give NaN
The 2nd order derivative of ELU gives off NaN about once out of hundreds of iterations.  Then the entire network will instantly contaminated with NaN.

I tracked the cause and the following code in backward() of class ELUGrad in chainer/chainer/functions/activation/elu.py is the cause
```
        if 1 in indexes:
            ret.append(ggxgx / gy)
```

It is natural that this division will give NaN if some element of gy is zero.  Zero will occur when the single-precision floating point subtraction underflows.

For your information, I am circumventing the issue by making minor modification to elu.py as in the attached file, which uses the same way to calculate derivative as the mathematical functions like F.exp.
[elu.py.txt](https://github.com/chainer/chainer/files/1683829/elu.py.txt)


How to reproduce:
I am using Chainer 3.2.0, but that part of the ELU source code is not different in v4.0, therefore I think this issue persists over the versions.
```
>>> import chainer
>>> import numpy as np
>>> x = chainer.Variable(np.array([[0, 0]],dtype=np.float32))
>>> y = chainer.functions.elu(x)
>>> y
variable([[ 0.,  0.]])
>>> 
>>> y.grad = (np.array([[0, 1e-30]],dtype=np.float32))
>>> y.backward(enable_double_backprop=True)
>>> 
>>> x.grad_var.grad = np.array([[1, 1]],dtype=np.float32)
>>> x.grad_var.backward()
/home/mogami/.pyenv/versions/anaconda3-4.2.0/lib/python3.5/site-packages/chainer/functions/math/basic_math.py:322: RuntimeWarning: invalid value encountered in true_divide
  return utils.force_array(x[0] / x[1]),
>>> y.grad_var
variable([[  0.00000000e+00,   1.00000000e-30]])
>>> y.grad_var.grad
array([[ nan,   1.]], dtype=float32)
```
The first element is nan, though it should be 1.0 in this case.  This example may seem silly when considering ELU only, but having zero for some of elements often happens when dy is back propagated from somewhere else because of underflow.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `chainer/functions/activation/elu.py`
Content:
```
1 import numpy
2 
3 from chainer.backends import cuda
4 from chainer import function_node
5 from chainer.utils import type_check
6 
7 
8 class ELU(function_node.FunctionNode):
9 
10     """Exponential Linear Unit."""
11 
12     def __init__(self, alpha=1.0):
13         self.alpha = float(alpha)
14 
15     def check_type_forward(self, in_types):
16         type_check.expect(in_types.size() == 1)
17         x_type, = in_types
18 
19         type_check.expect(x_type.dtype.kind == 'f')
20 
21     def forward_cpu(self, x):
22         self.retain_inputs((0,))
23         y = x[0].copy()
24         neg_indices = x[0] < 0
25         y[neg_indices] = self.alpha * (numpy.exp(y[neg_indices]) - 1)
26         return y,
27 
28     def forward_gpu(self, x):
29         self.retain_inputs((0,))
30         y = cuda.elementwise(
31             'T x, T alpha', 'T y',
32             'y = x >= 0 ? x : (T)(alpha * (exp(x) - 1))',
33             'elu_fwd')(
34                 x[0], self.alpha)
35         return y,
36 
37     def backward(self, indexes, grad_outputs):
38         x, = self.get_retained_inputs()
39         gy, = grad_outputs
40         return ELUGrad(self.alpha).apply((x, gy))
41 
42 
43 class ELUGrad(function_node.FunctionNode):
44 
45     """Exponential Linear Unit gradient function."""
46 
47     def __init__(self, alpha):
48         self.alpha = alpha
49 
50     def check_type_forward(self, in_types):
51         type_check.expect(in_types.size() == 2)
52         type_check.expect(in_types[0].dtype.kind == 'f')
53         type_check.expect(in_types[1].dtype.kind == 'f')
54 
55     def forward_cpu(self, inputs):
56         x, gy = inputs
57         gx = gy.copy()
58         neg_indices = x < 0
59         gx[neg_indices] *= self.alpha * numpy.exp(x[neg_indices])
60         self.retain_inputs((0, 1))
61         self.retain_outputs((0,))
62         return gx,
63 
64     def forward_gpu(self, inputs):
65         x, gy = inputs
66         gx = cuda.elementwise(
67             'T x, T gy, T alpha', 'T gx',
68             'gx = x >= 0 ? gy : (T)(gy * alpha * exp(x))',
69             'elu_bwd')(
70                 x, gy, self.alpha)
71         self.retain_inputs((0, 1))
72         self.retain_outputs((0,))
73         return gx,
74 
75     def backward(self, indexes, grad_outputs):
76         x, gy = self.get_retained_inputs()
77         gx, = self.get_retained_outputs()
78         ggx, = grad_outputs
79         ggxgx = ggx * gx
80 
81         ret = []
82         if 0 in indexes:
83             ret.append(ggxgx * (x.data < 0))
84         if 1 in indexes:
85             ret.append(ggxgx / gy)
86 
87         return ret
88 
89 
90 def elu(x, alpha=1.0):
91     """Exponential Linear Unit function.
92 
93     For a parameter :math:`\\alpha`, it is expressed as
94 
95     .. math::
96         f(x) = \\left \\{ \\begin{array}{ll}
97         x & {\\rm if}~ x \\ge 0 \\\\
98         \\alpha (\\exp(x) - 1) & {\\rm if}~ x < 0,
99         \\end{array} \\right.
100 
101     See: https://arxiv.org/abs/1511.07289
102 
103     Args:
104         x (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
105         :class:`cupy.ndarray`):
106             Input variable. A :math:`(s_1, s_2, ..., s_N)`-shaped float array.
107         alpha (float): Parameter :math:`\\alpha`. Default is 1.0.
108 
109     Returns:
110         ~chainer.Variable: Output variable. A
111         :math:`(s_1, s_2, ..., s_N)`-shaped float array.
112 
113     .. admonition:: Example
114 
115         >>> x = np.array([[-1, 0], [2, -3]], 'f')
116         >>> x
117         array([[-1.,  0.],
118                [ 2., -3.]], dtype=float32)
119         >>> y = F.elu(x, alpha=1.)
120         >>> y.data
121         array([[-0.63212055,  0.        ],
122                [ 2.        , -0.95021296]], dtype=float32)
123 
124     """
125     return ELU(alpha=alpha).apply((x,))[0]
126
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/chainer/functions/activation/elu.py b/chainer/functions/activation/elu.py
--- a/chainer/functions/activation/elu.py
+++ b/chainer/functions/activation/elu.py
@@ -22,14 +22,14 @@
         self.retain_inputs((0,))
         y = x[0].copy()
         neg_indices = x[0] < 0
-        y[neg_indices] = self.alpha * (numpy.exp(y[neg_indices]) - 1)
+        y[neg_indices] = self.alpha * (numpy.expm1(y[neg_indices]))
         return y,
 
     def forward_gpu(self, x):
         self.retain_inputs((0,))
         y = cuda.elementwise(
             'T x, T alpha', 'T y',
-            'y = x >= 0 ? x : (T)(alpha * (exp(x) - 1))',
+            'y = x >= 0 ? x : (T)(alpha * expm1(x))',
             'elu_fwd')(
                 x[0], self.alpha)
         return y,
@@ -37,7 +37,7 @@
     def backward(self, indexes, grad_outputs):
         x, = self.get_retained_inputs()
         gy, = grad_outputs
-        return ELUGrad(self.alpha).apply((x, gy))
+        return ELUGrad(self.alpha).apply((x,))[0] * gy,
 
 
 class ELUGrad(function_node.FunctionNode):
@@ -48,43 +48,34 @@
         self.alpha = alpha
 
     def check_type_forward(self, in_types):
-        type_check.expect(in_types.size() == 2)
+        type_check.expect(in_types.size() == 1)
         type_check.expect(in_types[0].dtype.kind == 'f')
-        type_check.expect(in_types[1].dtype.kind == 'f')
 
     def forward_cpu(self, inputs):
-        x, gy = inputs
-        gx = gy.copy()
+        x, = inputs
+        gx = numpy.ones_like(x)
         neg_indices = x < 0
         gx[neg_indices] *= self.alpha * numpy.exp(x[neg_indices])
-        self.retain_inputs((0, 1))
+        self.retain_inputs((0,))
         self.retain_outputs((0,))
         return gx,
 
     def forward_gpu(self, inputs):
-        x, gy = inputs
+        x, = inputs
         gx = cuda.elementwise(
-            'T x, T gy, T alpha', 'T gx',
-            'gx = x >= 0 ? gy : (T)(gy * alpha * exp(x))',
+            'T x, T alpha', 'T gx',
+            'gx = x >= 0 ? (T)1 : (T)(alpha * exp(x))',
             'elu_bwd')(
-                x, gy, self.alpha)
-        self.retain_inputs((0, 1))
+                x, self.alpha)
+        self.retain_inputs((0,))
         self.retain_outputs((0,))
         return gx,
 
     def backward(self, indexes, grad_outputs):
-        x, gy = self.get_retained_inputs()
+        x, = self.get_retained_inputs()
         gx, = self.get_retained_outputs()
         ggx, = grad_outputs
-        ggxgx = ggx * gx
-
-        ret = []
-        if 0 in indexes:
-            ret.append(ggxgx * (x.data < 0))
-        if 1 in indexes:
-            ret.append(ggxgx / gy)
-
-        return ret
+        return ggx * gx * (x.data < 0),
 
 
 def elu(x, alpha=1.0):

{"golden_diff": "diff --git a/chainer/functions/activation/elu.py b/chainer/functions/activation/elu.py\n--- a/chainer/functions/activation/elu.py\n+++ b/chainer/functions/activation/elu.py\n@@ -22,14 +22,14 @@\n         self.retain_inputs((0,))\n         y = x[0].copy()\n         neg_indices = x[0] < 0\n-        y[neg_indices] = self.alpha * (numpy.exp(y[neg_indices]) - 1)\n+        y[neg_indices] = self.alpha * (numpy.expm1(y[neg_indices]))\n         return y,\n \n     def forward_gpu(self, x):\n         self.retain_inputs((0,))\n         y = cuda.elementwise(\n             'T x, T alpha', 'T y',\n-            'y = x >= 0 ? x : (T)(alpha * (exp(x) - 1))',\n+            'y = x >= 0 ? x : (T)(alpha * expm1(x))',\n             'elu_fwd')(\n                 x[0], self.alpha)\n         return y,\n@@ -37,7 +37,7 @@\n     def backward(self, indexes, grad_outputs):\n         x, = self.get_retained_inputs()\n         gy, = grad_outputs\n-        return ELUGrad(self.alpha).apply((x, gy))\n+        return ELUGrad(self.alpha).apply((x,))[0] * gy,\n \n \n class ELUGrad(function_node.FunctionNode):\n@@ -48,43 +48,34 @@\n         self.alpha = alpha\n \n     def check_type_forward(self, in_types):\n-        type_check.expect(in_types.size() == 2)\n+        type_check.expect(in_types.size() == 1)\n         type_check.expect(in_types[0].dtype.kind == 'f')\n-        type_check.expect(in_types[1].dtype.kind == 'f')\n \n     def forward_cpu(self, inputs):\n-        x, gy = inputs\n-        gx = gy.copy()\n+        x, = inputs\n+        gx = numpy.ones_like(x)\n         neg_indices = x < 0\n         gx[neg_indices] *= self.alpha * numpy.exp(x[neg_indices])\n-        self.retain_inputs((0, 1))\n+        self.retain_inputs((0,))\n         self.retain_outputs((0,))\n         return gx,\n \n     def forward_gpu(self, inputs):\n-        x, gy = inputs\n+        x, = inputs\n         gx = cuda.elementwise(\n-            'T x, T gy, T alpha', 'T gx',\n-            'gx = x >= 0 ? gy : (T)(gy * alpha * exp(x))',\n+            'T x, T alpha', 'T gx',\n+            'gx = x >= 0 ? (T)1 : (T)(alpha * exp(x))',\n             'elu_bwd')(\n-                x, gy, self.alpha)\n-        self.retain_inputs((0, 1))\n+                x, self.alpha)\n+        self.retain_inputs((0,))\n         self.retain_outputs((0,))\n         return gx,\n \n     def backward(self, indexes, grad_outputs):\n-        x, gy = self.get_retained_inputs()\n+        x, = self.get_retained_inputs()\n         gx, = self.get_retained_outputs()\n         ggx, = grad_outputs\n-        ggxgx = ggx * gx\n-\n-        ret = []\n-        if 0 in indexes:\n-            ret.append(ggxgx * (x.data < 0))\n-        if 1 in indexes:\n-            ret.append(ggxgx / gy)\n-\n-        return ret\n+        return ggx * gx * (x.data < 0),\n \n \n def elu(x, alpha=1.0):\n", "issue": "2nd order derivative of ELU should not give NaN\nThe 2nd order derivative of ELU gives off NaN about once out of hundreds of iterations.  Then the entire network will instantly contaminated with NaN.\r\n\r\nI tracked the cause and the following code in backward() of class ELUGrad in chainer/chainer/functions/activation/elu.py is the cause\r\n```\r\n        if 1 in indexes:\r\n            ret.append(ggxgx / gy)\r\n```\r\n\r\nIt is natural that this division will give NaN if some element of gy is zero.  Zero will occur when the single-precision floating point subtraction underflows.\r\n\r\nFor your information, I am circumventing the issue by making minor modification to elu.py as in the attached file, which uses the same way to calculate derivative as the mathematical functions like F.exp.\r\n[elu.py.txt](https://github.com/chainer/chainer/files/1683829/elu.py.txt)\r\n\r\n\r\nHow to reproduce:\r\nI am using Chainer 3.2.0, but that part of the ELU source code is not different in v4.0, therefore I think this issue persists over the versions.\r\n```\r\n>>> import chainer\r\n>>> import numpy as np\r\n>>> x = chainer.Variable(np.array([[0, 0]],dtype=np.float32))\r\n>>> y = chainer.functions.elu(x)\r\n>>> y\r\nvariable([[ 0.,  0.]])\r\n>>> \r\n>>> y.grad = (np.array([[0, 1e-30]],dtype=np.float32))\r\n>>> y.backward(enable_double_backprop=True)\r\n>>> \r\n>>> x.grad_var.grad = np.array([[1, 1]],dtype=np.float32)\r\n>>> x.grad_var.backward()\r\n/home/mogami/.pyenv/versions/anaconda3-4.2.0/lib/python3.5/site-packages/chainer/functions/math/basic_math.py:322: RuntimeWarning: invalid value encountered in true_divide\r\n  return utils.force_array(x[0] / x[1]),\r\n>>> y.grad_var\r\nvariable([[  0.00000000e+00,   1.00000000e-30]])\r\n>>> y.grad_var.grad\r\narray([[ nan,   1.]], dtype=float32)\r\n```\r\nThe first element is nan, though it should be 1.0 in this case.  This example may seem silly when considering ELU only, but having zero for some of elements often happens when dy is back propagated from somewhere else because of underflow.\n", "before_files": [{"content": "import numpy\n\nfrom chainer.backends import cuda\nfrom chainer import function_node\nfrom chainer.utils import type_check\n\n\nclass ELU(function_node.FunctionNode):\n\n    \"\"\"Exponential Linear Unit.\"\"\"\n\n    def __init__(self, alpha=1.0):\n        self.alpha = float(alpha)\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 1)\n        x_type, = in_types\n\n        type_check.expect(x_type.dtype.kind == 'f')\n\n    def forward_cpu(self, x):\n        self.retain_inputs((0,))\n        y = x[0].copy()\n        neg_indices = x[0] < 0\n        y[neg_indices] = self.alpha * (numpy.exp(y[neg_indices]) - 1)\n        return y,\n\n    def forward_gpu(self, x):\n        self.retain_inputs((0,))\n        y = cuda.elementwise(\n            'T x, T alpha', 'T y',\n            'y = x >= 0 ? x : (T)(alpha * (exp(x) - 1))',\n            'elu_fwd')(\n                x[0], self.alpha)\n        return y,\n\n    def backward(self, indexes, grad_outputs):\n        x, = self.get_retained_inputs()\n        gy, = grad_outputs\n        return ELUGrad(self.alpha).apply((x, gy))\n\n\nclass ELUGrad(function_node.FunctionNode):\n\n    \"\"\"Exponential Linear Unit gradient function.\"\"\"\n\n    def __init__(self, alpha):\n        self.alpha = alpha\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 2)\n        type_check.expect(in_types[0].dtype.kind == 'f')\n        type_check.expect(in_types[1].dtype.kind == 'f')\n\n    def forward_cpu(self, inputs):\n        x, gy = inputs\n        gx = gy.copy()\n        neg_indices = x < 0\n        gx[neg_indices] *= self.alpha * numpy.exp(x[neg_indices])\n        self.retain_inputs((0, 1))\n        self.retain_outputs((0,))\n        return gx,\n\n    def forward_gpu(self, inputs):\n        x, gy = inputs\n        gx = cuda.elementwise(\n            'T x, T gy, T alpha', 'T gx',\n            'gx = x >= 0 ? gy : (T)(gy * alpha * exp(x))',\n            'elu_bwd')(\n                x, gy, self.alpha)\n        self.retain_inputs((0, 1))\n        self.retain_outputs((0,))\n        return gx,\n\n    def backward(self, indexes, grad_outputs):\n        x, gy = self.get_retained_inputs()\n        gx, = self.get_retained_outputs()\n        ggx, = grad_outputs\n        ggxgx = ggx * gx\n\n        ret = []\n        if 0 in indexes:\n            ret.append(ggxgx * (x.data < 0))\n        if 1 in indexes:\n            ret.append(ggxgx / gy)\n\n        return ret\n\n\ndef elu(x, alpha=1.0):\n    \"\"\"Exponential Linear Unit function.\n\n    For a parameter :math:`\\\\alpha`, it is expressed as\n\n    .. math::\n        f(x) = \\\\left \\\\{ \\\\begin{array}{ll}\n        x & {\\\\rm if}~ x \\\\ge 0 \\\\\\\\\n        \\\\alpha (\\\\exp(x) - 1) & {\\\\rm if}~ x < 0,\n        \\\\end{array} \\\\right.\n\n    See: https://arxiv.org/abs/1511.07289\n\n    Args:\n        x (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \\\n        :class:`cupy.ndarray`):\n            Input variable. A :math:`(s_1, s_2, ..., s_N)`-shaped float array.\n        alpha (float): Parameter :math:`\\\\alpha`. Default is 1.0.\n\n    Returns:\n        ~chainer.Variable: Output variable. A\n        :math:`(s_1, s_2, ..., s_N)`-shaped float array.\n\n    .. admonition:: Example\n\n        >>> x = np.array([[-1, 0], [2, -3]], 'f')\n        >>> x\n        array([[-1.,  0.],\n               [ 2., -3.]], dtype=float32)\n        >>> y = F.elu(x, alpha=1.)\n        >>> y.data\n        array([[-0.63212055,  0.        ],\n               [ 2.        , -0.95021296]], dtype=float32)\n\n    \"\"\"\n    return ELU(alpha=alpha).apply((x,))[0]\n", "path": "chainer/functions/activation/elu.py"}], "after_files": [{"content": "import numpy\n\nfrom chainer.backends import cuda\nfrom chainer import function_node\nfrom chainer.utils import type_check\n\n\nclass ELU(function_node.FunctionNode):\n\n    \"\"\"Exponential Linear Unit.\"\"\"\n\n    def __init__(self, alpha=1.0):\n        self.alpha = float(alpha)\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 1)\n        x_type, = in_types\n\n        type_check.expect(x_type.dtype.kind == 'f')\n\n    def forward_cpu(self, x):\n        self.retain_inputs((0,))\n        y = x[0].copy()\n        neg_indices = x[0] < 0\n        y[neg_indices] = self.alpha * (numpy.expm1(y[neg_indices]))\n        return y,\n\n    def forward_gpu(self, x):\n        self.retain_inputs((0,))\n        y = cuda.elementwise(\n            'T x, T alpha', 'T y',\n            'y = x >= 0 ? x : (T)(alpha * expm1(x))',\n            'elu_fwd')(\n                x[0], self.alpha)\n        return y,\n\n    def backward(self, indexes, grad_outputs):\n        x, = self.get_retained_inputs()\n        gy, = grad_outputs\n        return ELUGrad(self.alpha).apply((x,))[0] * gy,\n\n\nclass ELUGrad(function_node.FunctionNode):\n\n    \"\"\"Exponential Linear Unit gradient function.\"\"\"\n\n    def __init__(self, alpha):\n        self.alpha = alpha\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 1)\n        type_check.expect(in_types[0].dtype.kind == 'f')\n\n    def forward_cpu(self, inputs):\n        x, = inputs\n        gx = numpy.ones_like(x)\n        neg_indices = x < 0\n        gx[neg_indices] *= self.alpha * numpy.exp(x[neg_indices])\n        self.retain_inputs((0,))\n        self.retain_outputs((0,))\n        return gx,\n\n    def forward_gpu(self, inputs):\n        x, = inputs\n        gx = cuda.elementwise(\n            'T x, T alpha', 'T gx',\n            'gx = x >= 0 ? (T)1 : (T)(alpha * exp(x))',\n            'elu_bwd')(\n                x, self.alpha)\n        self.retain_inputs((0,))\n        self.retain_outputs((0,))\n        return gx,\n\n    def backward(self, indexes, grad_outputs):\n        x, = self.get_retained_inputs()\n        gx, = self.get_retained_outputs()\n        ggx, = grad_outputs\n        return ggx * gx * (x.data < 0),\n\n\ndef elu(x, alpha=1.0):\n    \"\"\"Exponential Linear Unit function.\n\n    For a parameter :math:`\\\\alpha`, it is expressed as\n\n    .. math::\n        f(x) = \\\\left \\\\{ \\\\begin{array}{ll}\n        x & {\\\\rm if}~ x \\\\ge 0 \\\\\\\\\n        \\\\alpha (\\\\exp(x) - 1) & {\\\\rm if}~ x < 0,\n        \\\\end{array} \\\\right.\n\n    See: https://arxiv.org/abs/1511.07289\n\n    Args:\n        x (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \\\n        :class:`cupy.ndarray`):\n            Input variable. A :math:`(s_1, s_2, ..., s_N)`-shaped float array.\n        alpha (float): Parameter :math:`\\\\alpha`. Default is 1.0.\n\n    Returns:\n        ~chainer.Variable: Output variable. A\n        :math:`(s_1, s_2, ..., s_N)`-shaped float array.\n\n    .. admonition:: Example\n\n        >>> x = np.array([[-1, 0], [2, -3]], 'f')\n        >>> x\n        array([[-1.,  0.],\n               [ 2., -3.]], dtype=float32)\n        >>> y = F.elu(x, alpha=1.)\n        >>> y.data\n        array([[-0.63212055,  0.        ],\n               [ 2.        , -0.95021296]], dtype=float32)\n\n    \"\"\"\n    return ELU(alpha=alpha).apply((x,))[0]\n", "path": "chainer/functions/activation/elu.py"}]}