We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
ignore_label in Softmax.forward_gpu does not work when it is not -1
`F.SoftMaxCrossEntropy` can specify `ignore_label` in its `__init__` method. But -1 is hard-coded in its `forward_gpu` and `backward_gpu` methods ([here](https://github.com/pfnet/chainer/blob/master/chainer/functions/loss/softmax_cross_entropy.py#L113) and [here](https://github.com/pfnet/chainer/blob/master/chainer/functions/loss/softmax_cross_entropy.py#L195)). Therefore it does not calculate loss value if `ignore_label` is other than `-1`.

This bug is reported by @musyoku. Thank you!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `chainer/functions/loss/softmax_cross_entropy.py`
Content:
```
1 import numpy
2 import six
3 
4 import chainer
5 from chainer import cuda
6 from chainer import function
7 from chainer.functions.activation import log_softmax
8 from chainer.utils import type_check
9 
10 
11 class SoftmaxCrossEntropy(function.Function):
12 
13     """Softmax activation followed by a cross entropy loss."""
14 
15     normalize = True
16 
17     def __init__(self, use_cudnn=True, normalize=True, cache_score=True,
18                  class_weight=None, ignore_label=-1, reduce='mean'):
19         self.use_cudnn = use_cudnn
20         self.normalize = normalize
21         self.cache_score = cache_score
22         self.class_weight = class_weight
23         if class_weight is not None:
24             if self.class_weight.ndim != 1:
25                 raise ValueError('class_weight.ndim should be 1')
26             if self.class_weight.dtype.kind != 'f':
27                 raise ValueError('The dtype of class_weight should be \'f\'')
28             if isinstance(self.class_weight, chainer.Variable):
29                 raise ValueError('class_weight should be a numpy.ndarray or '
30                                  'cupy.ndarray, not a chainer.Variable')
31         self.ignore_label = ignore_label
32         if reduce not in ('mean', 'no'):
33             raise ValueError(
34                 "only 'mean' and 'no' are valid for 'reduce', but '%s' is "
35                 'given' % reduce)
36         self.reduce = reduce
37 
38     def check_type_forward(self, in_types):
39         type_check.expect(in_types.size() == 2)
40         x_type, t_type = in_types
41 
42         type_check.expect(
43             x_type.dtype.kind == 'f',
44             t_type.dtype == numpy.int32,
45             t_type.ndim == x_type.ndim - 1,
46 
47             x_type.shape[0] == t_type.shape[0],
48             x_type.shape[2:] == t_type.shape[1:],
49         )
50 
51     def _check_input_values(self, x, t):
52         if not (((0 <= t) &
53                  (t < x.shape[1])) |
54                 (t == self.ignore_label)).all():
55             msg = ('Each label `t` need to satisfy '
56                    '`0 <= t < x.shape[1] or t == %d`' % self.ignore_label)
57             raise ValueError(msg)
58 
59     def forward_cpu(self, inputs):
60         x, t = inputs
61         if chainer.is_debug():
62             self._check_input_values(x, t)
63 
64         log_y = log_softmax._log_softmax(x, self.use_cudnn)
65         if self.cache_score:
66             self.y = numpy.exp(log_y)
67         if self.class_weight is not None:
68             shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
69             log_y *= numpy.broadcast_to(
70                 self.class_weight.reshape(shape), x.shape)
71         log_yd = numpy.rollaxis(log_y, 1)
72         log_yd = log_yd.reshape(len(log_yd), -1)
73         log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]
74 
75         log_p *= (t.ravel() != self.ignore_label)
76         if self.reduce == 'mean':
77             # deal with the case where the SoftmaxCrossEntropy is
78             # unpickled from the old version
79             if self.normalize:
80                 count = (t != self.ignore_label).sum()
81             else:
82                 count = len(x)
83             self._coeff = 1.0 / max(count, 1)
84 
85             y = log_p.sum(keepdims=True) * (-self._coeff)
86             return y.reshape(()),
87         else:
88             return -log_p.reshape(t.shape),
89 
90     def forward_gpu(self, inputs):
91         cupy = cuda.cupy
92         x, t = inputs
93         if chainer.is_debug():
94             self._check_input_values(x, t)
95 
96         log_y = log_softmax._log_softmax(x, self.use_cudnn)
97         if self.cache_score:
98             self.y = cupy.exp(log_y)
99         if self.class_weight is not None:
100             shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
101             log_y *= cupy.broadcast_to(
102                 self.class_weight.reshape(shape), x.shape)
103         if self.normalize:
104             coeff = cupy.maximum(1, (t != self.ignore_label).sum())
105         else:
106             coeff = max(1, len(t))
107         self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)
108 
109         log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
110         if self.reduce == 'mean':
111             ret = cuda.reduce(
112                 'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',
113                 't == -1 ? T(0) : log_y[_j * n_channel + t]',
114                 'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
115             )(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)
116         else:
117             ret = cuda.elementwise(
118                 'S t, raw T log_y, int32 n_channel, T ignore', 'T out',
119                 '''
120                 if (t == ignore) {
121                   out = 0;
122                 } else {
123                   out = -log_y[i * n_channel + t];
124                 }
125                 ''',
126                 'softmax_crossent_no_reduce_fwd'
127             )(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)
128             ret = ret.reshape(t.shape)
129         return ret,
130 
131     def backward_cpu(self, inputs, grad_outputs):
132         x, t = inputs
133         gloss = grad_outputs[0]
134         if hasattr(self, 'y'):
135             y = self.y.copy()
136         else:
137             y = log_softmax._log_softmax(x, self.use_cudnn)
138             numpy.exp(y, out=y)
139         if y.ndim == 2:
140             gx = y
141             gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1
142             if self.class_weight is not None:
143                 shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
144                 c = numpy.broadcast_to(
145                     self.class_weight.reshape(shape), x.shape)
146                 c = c[numpy.arange(len(t)), numpy.maximum(t, 0)]
147                 gx *= numpy.broadcast_to(numpy.expand_dims(c, 1), gx.shape)
148             gx *= (t != self.ignore_label).reshape((len(t), 1))
149         else:
150             # in the case where y.ndim is higher than 2,
151             # we think that a current implementation is inefficient
152             # because it yields two provisional arrays for indexing.
153             n_unit = t.size // len(t)
154             gx = y.reshape(y.shape[0], y.shape[1], -1)
155             fst_index = numpy.arange(t.size) // n_unit
156             trd_index = numpy.arange(t.size) % n_unit
157             gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1
158             if self.class_weight is not None:
159                 shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]
160                 c = numpy.broadcast_to(
161                     self.class_weight.reshape(shape), x.shape)
162                 c = c.reshape(gx.shape)
163                 c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index]
164                 c = c.reshape(y.shape[0], 1, -1)
165                 gx *= numpy.broadcast_to(c, gx.shape)
166             gx *= (t != self.ignore_label).reshape((len(t), 1, -1))
167             gx = gx.reshape(y.shape)
168         if self.reduce == 'mean':
169             gx *= gloss * self._coeff
170         else:
171             gx *= gloss[:, None]
172         return gx, None
173 
174     def backward_gpu(self, inputs, grad_outputs):
175         cupy = cuda.cupy
176         x, t = inputs
177         if hasattr(self, 'y'):
178             y = self.y
179         else:
180             y = log_softmax._log_softmax(x, self.use_cudnn)
181             cupy.exp(y, out=y)
182         gloss = grad_outputs[0]
183         n_unit = t.size // len(t)
184         if self.reduce == 'mean':
185             coeff = gloss * self._coeff
186         else:
187             coeff = gloss[:, None, ...]
188 
189         if self.class_weight is None:
190             gx = cuda.elementwise(
191                 'T y, S t, T coeff, S n_channel, S n_unit',
192                 'T gx',
193                 '''
194                     const int c = (i / n_unit % n_channel);
195                     gx = t == -1 ? 0 : coeff * (y - (c == t));
196                 ''',
197                 'softmax_crossent_bwd')(
198                     y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit)
199         else:
200             gx = cuda.elementwise(
201                 'T y, raw T w, S t, T coeff, S n_channel, S n_unit',
202                 'T gx',
203                 '''
204                     const int c = (i / n_unit % n_channel);
205                     gx = t == -1 ? 0 : coeff * (y - (c == t)) * w[t];
206                 ''',
207                 'softmax_crossent_weight_bwd')(
208                     y, self.class_weight, cupy.expand_dims(t, 1), coeff,
209                     x.shape[1], n_unit)
210 
211         return gx, None
212 
213 
214 def softmax_cross_entropy(
215         x, t, use_cudnn=True, normalize=True, cache_score=True,
216         class_weight=None, ignore_label=-1, reduce='mean'):
217     """Computes cross entropy loss for pre-softmax activations.
218 
219     Args:
220         x (~chainer.Variable): Variable holding a multidimensional array whose
221             element indicates unnormalized log probability: the first axis of
222             the variable represents the number of samples, and the second axis
223             represents the number of classes. While this function computes
224             a usual softmax cross entropy if the number of dimensions is equal
225             to 2, it computes a cross entropy of the replicated softmax if the
226             number of dimensions is greater than 2.
227         t (~chainer.Variable): Variable holding an int32 vector of ground truth
228             labels. If ``t[i] == ignore_label``, corresponding ``x[i]`` is
229             ignored.
230         normalize (bool): If ``True``, this function normalizes the cross
231             entropy loss across all instances. If ``False``, it only
232             normalizes along a batch size.
233         cache_score (bool): When it is ``True``, the function stores result
234             of forward computation to use it on backward computation. It
235             reduces computational cost though consumes more memory.
236         class_weight (~numpy.ndarray or ~chainer.cuda.cupy.ndarray): An array
237             that contains constant weights that will be multiplied with the
238             loss values along with the second dimension. The shape of this
239             array should be ``(x.shape[1],)``. If this is not ``None``, each
240             class weight ``class_weight[i]`` is actually multiplied to
241             ``y[:, i]`` that is the corresponding log-softmax output of ``x``
242             and has the same shape as ``x`` before calculating the actual loss
243             value.
244         ignore_label (int): Label value you want to ignore. Its default value
245             is ``-1``. See description of the argument `t`.
246         reduce (str): A string that determines whether to reduce the loss
247             values. If it is ``'mean'``, it computes the sum of the individual
248             cross entropy and normalize it according to ``normalize`` option.
249             If it is ``'no'``, this function computes cross entropy for each
250             instance and does not normalize it (``normalize`` option is
251             ignored). In this case, the loss value of the ignored instance,
252             which has ``ignore_label`` as its target value, is set to ``0``.
253 
254     Returns:
255         Variable: A variable holding a scalar array of the cross entropy loss.
256         If ``reduce`` is ``'mean'``, it is a scalar array.
257         If ``reduce`` is ``'no'``, the shape is same as that of ``x``.
258 
259     .. note::
260 
261        This function is differentiable only by ``x``.
262 
263     """
264 
265     return SoftmaxCrossEntropy(
266         use_cudnn, normalize, cache_score, class_weight, ignore_label, reduce)(
267             x, t)
268
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/chainer/functions/loss/softmax_cross_entropy.py b/chainer/functions/loss/softmax_cross_entropy.py
--- a/chainer/functions/loss/softmax_cross_entropy.py
+++ b/chainer/functions/loss/softmax_cross_entropy.py
@@ -109,10 +109,13 @@
         log_y = cupy.rollaxis(log_y, 1, log_y.ndim)
         if self.reduce == 'mean':
             ret = cuda.reduce(
-                'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',
-                't == -1 ? T(0) : log_y[_j * n_channel + t]',
+                'S t, raw T log_y, int32 n_channel, raw T coeff, '
+                'S ignore_label',
+                'T out',
+                't == ignore_label ? T(0) : log_y[_j * n_channel + t]',
                 'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'
-            )(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)
+            )(t, log_y.reduced_view(), log_y.shape[-1],
+              self._coeff, self.ignore_label)
         else:
             ret = cuda.elementwise(
                 'S t, raw T log_y, int32 n_channel, T ignore', 'T out',
@@ -188,25 +191,27 @@
 
         if self.class_weight is None:
             gx = cuda.elementwise(
-                'T y, S t, T coeff, S n_channel, S n_unit',
+                'T y, S t, T coeff, S n_channel, S n_unit, S ignore_label',
                 'T gx',
                 '''
                     const int c = (i / n_unit % n_channel);
-                    gx = t == -1 ? 0 : coeff * (y - (c == t));
+                    gx = t == ignore_label ? 0 : coeff * (y - (c == t));
                 ''',
                 'softmax_crossent_bwd')(
-                    y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit)
+                    y, cupy.expand_dims(t, 1), coeff, x.shape[1],
+                    n_unit, self.ignore_label)
         else:
             gx = cuda.elementwise(
-                'T y, raw T w, S t, T coeff, S n_channel, S n_unit',
+                'T y, raw T w, S t, T coeff, S n_channel, S n_unit, '
+                'S ignore_label',
                 'T gx',
                 '''
                     const int c = (i / n_unit % n_channel);
-                    gx = t == -1 ? 0 : coeff * (y - (c == t)) * w[t];
+                    gx = t == ignore_label ? 0 : coeff * (y - (c == t)) * w[t];
                 ''',
                 'softmax_crossent_weight_bwd')(
                     y, self.class_weight, cupy.expand_dims(t, 1), coeff,
-                    x.shape[1], n_unit)
+                    x.shape[1], n_unit, self.ignore_label)
 
         return gx, None

{"golden_diff": "diff --git a/chainer/functions/loss/softmax_cross_entropy.py b/chainer/functions/loss/softmax_cross_entropy.py\n--- a/chainer/functions/loss/softmax_cross_entropy.py\n+++ b/chainer/functions/loss/softmax_cross_entropy.py\n@@ -109,10 +109,13 @@\n         log_y = cupy.rollaxis(log_y, 1, log_y.ndim)\n         if self.reduce == 'mean':\n             ret = cuda.reduce(\n-                'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',\n-                't == -1 ? T(0) : log_y[_j * n_channel + t]',\n+                'S t, raw T log_y, int32 n_channel, raw T coeff, '\n+                'S ignore_label',\n+                'T out',\n+                't == ignore_label ? T(0) : log_y[_j * n_channel + t]',\n                 'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'\n-            )(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)\n+            )(t, log_y.reduced_view(), log_y.shape[-1],\n+              self._coeff, self.ignore_label)\n         else:\n             ret = cuda.elementwise(\n                 'S t, raw T log_y, int32 n_channel, T ignore', 'T out',\n@@ -188,25 +191,27 @@\n \n         if self.class_weight is None:\n             gx = cuda.elementwise(\n-                'T y, S t, T coeff, S n_channel, S n_unit',\n+                'T y, S t, T coeff, S n_channel, S n_unit, S ignore_label',\n                 'T gx',\n                 '''\n                     const int c = (i / n_unit % n_channel);\n-                    gx = t == -1 ? 0 : coeff * (y - (c == t));\n+                    gx = t == ignore_label ? 0 : coeff * (y - (c == t));\n                 ''',\n                 'softmax_crossent_bwd')(\n-                    y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit)\n+                    y, cupy.expand_dims(t, 1), coeff, x.shape[1],\n+                    n_unit, self.ignore_label)\n         else:\n             gx = cuda.elementwise(\n-                'T y, raw T w, S t, T coeff, S n_channel, S n_unit',\n+                'T y, raw T w, S t, T coeff, S n_channel, S n_unit, '\n+                'S ignore_label',\n                 'T gx',\n                 '''\n                     const int c = (i / n_unit % n_channel);\n-                    gx = t == -1 ? 0 : coeff * (y - (c == t)) * w[t];\n+                    gx = t == ignore_label ? 0 : coeff * (y - (c == t)) * w[t];\n                 ''',\n                 'softmax_crossent_weight_bwd')(\n                     y, self.class_weight, cupy.expand_dims(t, 1), coeff,\n-                    x.shape[1], n_unit)\n+                    x.shape[1], n_unit, self.ignore_label)\n \n         return gx, None\n", "issue": "ignore_label in Softmax.forward_gpu does not work when it is not -1\n`F.SoftMaxCrossEntropy` can specify `ignore_label` in its `__init__` method. But -1 is hard-coded in its `forward_gpu` and `backward_gpu` methods ([here](https://github.com/pfnet/chainer/blob/master/chainer/functions/loss/softmax_cross_entropy.py#L113) and [here](https://github.com/pfnet/chainer/blob/master/chainer/functions/loss/softmax_cross_entropy.py#L195)). Therefore it does not calculate loss value if `ignore_label` is other than `-1`.\r\n\r\nThis bug is reported by @musyoku. Thank you!\n", "before_files": [{"content": "import numpy\nimport six\n\nimport chainer\nfrom chainer import cuda\nfrom chainer import function\nfrom chainer.functions.activation import log_softmax\nfrom chainer.utils import type_check\n\n\nclass SoftmaxCrossEntropy(function.Function):\n\n    \"\"\"Softmax activation followed by a cross entropy loss.\"\"\"\n\n    normalize = True\n\n    def __init__(self, use_cudnn=True, normalize=True, cache_score=True,\n                 class_weight=None, ignore_label=-1, reduce='mean'):\n        self.use_cudnn = use_cudnn\n        self.normalize = normalize\n        self.cache_score = cache_score\n        self.class_weight = class_weight\n        if class_weight is not None:\n            if self.class_weight.ndim != 1:\n                raise ValueError('class_weight.ndim should be 1')\n            if self.class_weight.dtype.kind != 'f':\n                raise ValueError('The dtype of class_weight should be \\'f\\'')\n            if isinstance(self.class_weight, chainer.Variable):\n                raise ValueError('class_weight should be a numpy.ndarray or '\n                                 'cupy.ndarray, not a chainer.Variable')\n        self.ignore_label = ignore_label\n        if reduce not in ('mean', 'no'):\n            raise ValueError(\n                \"only 'mean' and 'no' are valid for 'reduce', but '%s' is \"\n                'given' % reduce)\n        self.reduce = reduce\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 2)\n        x_type, t_type = in_types\n\n        type_check.expect(\n            x_type.dtype.kind == 'f',\n            t_type.dtype == numpy.int32,\n            t_type.ndim == x_type.ndim - 1,\n\n            x_type.shape[0] == t_type.shape[0],\n            x_type.shape[2:] == t_type.shape[1:],\n        )\n\n    def _check_input_values(self, x, t):\n        if not (((0 <= t) &\n                 (t < x.shape[1])) |\n                (t == self.ignore_label)).all():\n            msg = ('Each label `t` need to satisfy '\n                   '`0 <= t < x.shape[1] or t == %d`' % self.ignore_label)\n            raise ValueError(msg)\n\n    def forward_cpu(self, inputs):\n        x, t = inputs\n        if chainer.is_debug():\n            self._check_input_values(x, t)\n\n        log_y = log_softmax._log_softmax(x, self.use_cudnn)\n        if self.cache_score:\n            self.y = numpy.exp(log_y)\n        if self.class_weight is not None:\n            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n            log_y *= numpy.broadcast_to(\n                self.class_weight.reshape(shape), x.shape)\n        log_yd = numpy.rollaxis(log_y, 1)\n        log_yd = log_yd.reshape(len(log_yd), -1)\n        log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]\n\n        log_p *= (t.ravel() != self.ignore_label)\n        if self.reduce == 'mean':\n            # deal with the case where the SoftmaxCrossEntropy is\n            # unpickled from the old version\n            if self.normalize:\n                count = (t != self.ignore_label).sum()\n            else:\n                count = len(x)\n            self._coeff = 1.0 / max(count, 1)\n\n            y = log_p.sum(keepdims=True) * (-self._coeff)\n            return y.reshape(()),\n        else:\n            return -log_p.reshape(t.shape),\n\n    def forward_gpu(self, inputs):\n        cupy = cuda.cupy\n        x, t = inputs\n        if chainer.is_debug():\n            self._check_input_values(x, t)\n\n        log_y = log_softmax._log_softmax(x, self.use_cudnn)\n        if self.cache_score:\n            self.y = cupy.exp(log_y)\n        if self.class_weight is not None:\n            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n            log_y *= cupy.broadcast_to(\n                self.class_weight.reshape(shape), x.shape)\n        if self.normalize:\n            coeff = cupy.maximum(1, (t != self.ignore_label).sum())\n        else:\n            coeff = max(1, len(t))\n        self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)\n\n        log_y = cupy.rollaxis(log_y, 1, log_y.ndim)\n        if self.reduce == 'mean':\n            ret = cuda.reduce(\n                'S t, raw T log_y, int32 n_channel, raw T coeff', 'T out',\n                't == -1 ? T(0) : log_y[_j * n_channel + t]',\n                'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'\n            )(t, log_y.reduced_view(), log_y.shape[-1], self._coeff)\n        else:\n            ret = cuda.elementwise(\n                'S t, raw T log_y, int32 n_channel, T ignore', 'T out',\n                '''\n                if (t == ignore) {\n                  out = 0;\n                } else {\n                  out = -log_y[i * n_channel + t];\n                }\n                ''',\n                'softmax_crossent_no_reduce_fwd'\n            )(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)\n            ret = ret.reshape(t.shape)\n        return ret,\n\n    def backward_cpu(self, inputs, grad_outputs):\n        x, t = inputs\n        gloss = grad_outputs[0]\n        if hasattr(self, 'y'):\n            y = self.y.copy()\n        else:\n            y = log_softmax._log_softmax(x, self.use_cudnn)\n            numpy.exp(y, out=y)\n        if y.ndim == 2:\n            gx = y\n            gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1\n            if self.class_weight is not None:\n                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n                c = numpy.broadcast_to(\n                    self.class_weight.reshape(shape), x.shape)\n                c = c[numpy.arange(len(t)), numpy.maximum(t, 0)]\n                gx *= numpy.broadcast_to(numpy.expand_dims(c, 1), gx.shape)\n            gx *= (t != self.ignore_label).reshape((len(t), 1))\n        else:\n            # in the case where y.ndim is higher than 2,\n            # we think that a current implementation is inefficient\n            # because it yields two provisional arrays for indexing.\n            n_unit = t.size // len(t)\n            gx = y.reshape(y.shape[0], y.shape[1], -1)\n            fst_index = numpy.arange(t.size) // n_unit\n            trd_index = numpy.arange(t.size) % n_unit\n            gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1\n            if self.class_weight is not None:\n                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n                c = numpy.broadcast_to(\n                    self.class_weight.reshape(shape), x.shape)\n                c = c.reshape(gx.shape)\n                c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index]\n                c = c.reshape(y.shape[0], 1, -1)\n                gx *= numpy.broadcast_to(c, gx.shape)\n            gx *= (t != self.ignore_label).reshape((len(t), 1, -1))\n            gx = gx.reshape(y.shape)\n        if self.reduce == 'mean':\n            gx *= gloss * self._coeff\n        else:\n            gx *= gloss[:, None]\n        return gx, None\n\n    def backward_gpu(self, inputs, grad_outputs):\n        cupy = cuda.cupy\n        x, t = inputs\n        if hasattr(self, 'y'):\n            y = self.y\n        else:\n            y = log_softmax._log_softmax(x, self.use_cudnn)\n            cupy.exp(y, out=y)\n        gloss = grad_outputs[0]\n        n_unit = t.size // len(t)\n        if self.reduce == 'mean':\n            coeff = gloss * self._coeff\n        else:\n            coeff = gloss[:, None, ...]\n\n        if self.class_weight is None:\n            gx = cuda.elementwise(\n                'T y, S t, T coeff, S n_channel, S n_unit',\n                'T gx',\n                '''\n                    const int c = (i / n_unit % n_channel);\n                    gx = t == -1 ? 0 : coeff * (y - (c == t));\n                ''',\n                'softmax_crossent_bwd')(\n                    y, cupy.expand_dims(t, 1), coeff, x.shape[1], n_unit)\n        else:\n            gx = cuda.elementwise(\n                'T y, raw T w, S t, T coeff, S n_channel, S n_unit',\n                'T gx',\n                '''\n                    const int c = (i / n_unit % n_channel);\n                    gx = t == -1 ? 0 : coeff * (y - (c == t)) * w[t];\n                ''',\n                'softmax_crossent_weight_bwd')(\n                    y, self.class_weight, cupy.expand_dims(t, 1), coeff,\n                    x.shape[1], n_unit)\n\n        return gx, None\n\n\ndef softmax_cross_entropy(\n        x, t, use_cudnn=True, normalize=True, cache_score=True,\n        class_weight=None, ignore_label=-1, reduce='mean'):\n    \"\"\"Computes cross entropy loss for pre-softmax activations.\n\n    Args:\n        x (~chainer.Variable): Variable holding a multidimensional array whose\n            element indicates unnormalized log probability: the first axis of\n            the variable represents the number of samples, and the second axis\n            represents the number of classes. While this function computes\n            a usual softmax cross entropy if the number of dimensions is equal\n            to 2, it computes a cross entropy of the replicated softmax if the\n            number of dimensions is greater than 2.\n        t (~chainer.Variable): Variable holding an int32 vector of ground truth\n            labels. If ``t[i] == ignore_label``, corresponding ``x[i]`` is\n            ignored.\n        normalize (bool): If ``True``, this function normalizes the cross\n            entropy loss across all instances. If ``False``, it only\n            normalizes along a batch size.\n        cache_score (bool): When it is ``True``, the function stores result\n            of forward computation to use it on backward computation. It\n            reduces computational cost though consumes more memory.\n        class_weight (~numpy.ndarray or ~chainer.cuda.cupy.ndarray): An array\n            that contains constant weights that will be multiplied with the\n            loss values along with the second dimension. The shape of this\n            array should be ``(x.shape[1],)``. If this is not ``None``, each\n            class weight ``class_weight[i]`` is actually multiplied to\n            ``y[:, i]`` that is the corresponding log-softmax output of ``x``\n            and has the same shape as ``x`` before calculating the actual loss\n            value.\n        ignore_label (int): Label value you want to ignore. Its default value\n            is ``-1``. See description of the argument `t`.\n        reduce (str): A string that determines whether to reduce the loss\n            values. If it is ``'mean'``, it computes the sum of the individual\n            cross entropy and normalize it according to ``normalize`` option.\n            If it is ``'no'``, this function computes cross entropy for each\n            instance and does not normalize it (``normalize`` option is\n            ignored). In this case, the loss value of the ignored instance,\n            which has ``ignore_label`` as its target value, is set to ``0``.\n\n    Returns:\n        Variable: A variable holding a scalar array of the cross entropy loss.\n        If ``reduce`` is ``'mean'``, it is a scalar array.\n        If ``reduce`` is ``'no'``, the shape is same as that of ``x``.\n\n    .. note::\n\n       This function is differentiable only by ``x``.\n\n    \"\"\"\n\n    return SoftmaxCrossEntropy(\n        use_cudnn, normalize, cache_score, class_weight, ignore_label, reduce)(\n            x, t)\n", "path": "chainer/functions/loss/softmax_cross_entropy.py"}], "after_files": [{"content": "import numpy\nimport six\n\nimport chainer\nfrom chainer import cuda\nfrom chainer import function\nfrom chainer.functions.activation import log_softmax\nfrom chainer.utils import type_check\n\n\nclass SoftmaxCrossEntropy(function.Function):\n\n    \"\"\"Softmax activation followed by a cross entropy loss.\"\"\"\n\n    normalize = True\n\n    def __init__(self, use_cudnn=True, normalize=True, cache_score=True,\n                 class_weight=None, ignore_label=-1, reduce='mean'):\n        self.use_cudnn = use_cudnn\n        self.normalize = normalize\n        self.cache_score = cache_score\n        self.class_weight = class_weight\n        if class_weight is not None:\n            if self.class_weight.ndim != 1:\n                raise ValueError('class_weight.ndim should be 1')\n            if self.class_weight.dtype.kind != 'f':\n                raise ValueError('The dtype of class_weight should be \\'f\\'')\n            if isinstance(self.class_weight, chainer.Variable):\n                raise ValueError('class_weight should be a numpy.ndarray or '\n                                 'cupy.ndarray, not a chainer.Variable')\n        self.ignore_label = ignore_label\n        if reduce not in ('mean', 'no'):\n            raise ValueError(\n                \"only 'mean' and 'no' are valid for 'reduce', but '%s' is \"\n                'given' % reduce)\n        self.reduce = reduce\n\n    def check_type_forward(self, in_types):\n        type_check.expect(in_types.size() == 2)\n        x_type, t_type = in_types\n\n        type_check.expect(\n            x_type.dtype.kind == 'f',\n            t_type.dtype == numpy.int32,\n            t_type.ndim == x_type.ndim - 1,\n\n            x_type.shape[0] == t_type.shape[0],\n            x_type.shape[2:] == t_type.shape[1:],\n        )\n\n    def _check_input_values(self, x, t):\n        if not (((0 <= t) &\n                 (t < x.shape[1])) |\n                (t == self.ignore_label)).all():\n            msg = ('Each label `t` need to satisfy '\n                   '`0 <= t < x.shape[1] or t == %d`' % self.ignore_label)\n            raise ValueError(msg)\n\n    def forward_cpu(self, inputs):\n        x, t = inputs\n        if chainer.is_debug():\n            self._check_input_values(x, t)\n\n        log_y = log_softmax._log_softmax(x, self.use_cudnn)\n        if self.cache_score:\n            self.y = numpy.exp(log_y)\n        if self.class_weight is not None:\n            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n            log_y *= numpy.broadcast_to(\n                self.class_weight.reshape(shape), x.shape)\n        log_yd = numpy.rollaxis(log_y, 1)\n        log_yd = log_yd.reshape(len(log_yd), -1)\n        log_p = log_yd[numpy.maximum(t.ravel(), 0), numpy.arange(t.size)]\n\n        log_p *= (t.ravel() != self.ignore_label)\n        if self.reduce == 'mean':\n            # deal with the case where the SoftmaxCrossEntropy is\n            # unpickled from the old version\n            if self.normalize:\n                count = (t != self.ignore_label).sum()\n            else:\n                count = len(x)\n            self._coeff = 1.0 / max(count, 1)\n\n            y = log_p.sum(keepdims=True) * (-self._coeff)\n            return y.reshape(()),\n        else:\n            return -log_p.reshape(t.shape),\n\n    def forward_gpu(self, inputs):\n        cupy = cuda.cupy\n        x, t = inputs\n        if chainer.is_debug():\n            self._check_input_values(x, t)\n\n        log_y = log_softmax._log_softmax(x, self.use_cudnn)\n        if self.cache_score:\n            self.y = cupy.exp(log_y)\n        if self.class_weight is not None:\n            shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n            log_y *= cupy.broadcast_to(\n                self.class_weight.reshape(shape), x.shape)\n        if self.normalize:\n            coeff = cupy.maximum(1, (t != self.ignore_label).sum())\n        else:\n            coeff = max(1, len(t))\n        self._coeff = cupy.divide(1.0, coeff, dtype=x.dtype)\n\n        log_y = cupy.rollaxis(log_y, 1, log_y.ndim)\n        if self.reduce == 'mean':\n            ret = cuda.reduce(\n                'S t, raw T log_y, int32 n_channel, raw T coeff, '\n                'S ignore_label',\n                'T out',\n                't == ignore_label ? T(0) : log_y[_j * n_channel + t]',\n                'a + b', 'out = a * -coeff[0]', '0', 'crossent_fwd'\n            )(t, log_y.reduced_view(), log_y.shape[-1],\n              self._coeff, self.ignore_label)\n        else:\n            ret = cuda.elementwise(\n                'S t, raw T log_y, int32 n_channel, T ignore', 'T out',\n                '''\n                if (t == ignore) {\n                  out = 0;\n                } else {\n                  out = -log_y[i * n_channel + t];\n                }\n                ''',\n                'softmax_crossent_no_reduce_fwd'\n            )(t, log_y.reduced_view(), log_y.shape[-1], self.ignore_label)\n            ret = ret.reshape(t.shape)\n        return ret,\n\n    def backward_cpu(self, inputs, grad_outputs):\n        x, t = inputs\n        gloss = grad_outputs[0]\n        if hasattr(self, 'y'):\n            y = self.y.copy()\n        else:\n            y = log_softmax._log_softmax(x, self.use_cudnn)\n            numpy.exp(y, out=y)\n        if y.ndim == 2:\n            gx = y\n            gx[numpy.arange(len(t)), numpy.maximum(t, 0)] -= 1\n            if self.class_weight is not None:\n                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n                c = numpy.broadcast_to(\n                    self.class_weight.reshape(shape), x.shape)\n                c = c[numpy.arange(len(t)), numpy.maximum(t, 0)]\n                gx *= numpy.broadcast_to(numpy.expand_dims(c, 1), gx.shape)\n            gx *= (t != self.ignore_label).reshape((len(t), 1))\n        else:\n            # in the case where y.ndim is higher than 2,\n            # we think that a current implementation is inefficient\n            # because it yields two provisional arrays for indexing.\n            n_unit = t.size // len(t)\n            gx = y.reshape(y.shape[0], y.shape[1], -1)\n            fst_index = numpy.arange(t.size) // n_unit\n            trd_index = numpy.arange(t.size) % n_unit\n            gx[fst_index, numpy.maximum(t.ravel(), 0), trd_index] -= 1\n            if self.class_weight is not None:\n                shape = [1 if d != 1 else -1 for d in six.moves.range(x.ndim)]\n                c = numpy.broadcast_to(\n                    self.class_weight.reshape(shape), x.shape)\n                c = c.reshape(gx.shape)\n                c = c[fst_index, numpy.maximum(t.ravel(), 0), trd_index]\n                c = c.reshape(y.shape[0], 1, -1)\n                gx *= numpy.broadcast_to(c, gx.shape)\n            gx *= (t != self.ignore_label).reshape((len(t), 1, -1))\n            gx = gx.reshape(y.shape)\n        if self.reduce == 'mean':\n            gx *= gloss * self._coeff\n        else:\n            gx *= gloss[:, None]\n        return gx, None\n\n    def backward_gpu(self, inputs, grad_outputs):\n        cupy = cuda.cupy\n        x, t = inputs\n        if hasattr(self, 'y'):\n            y = self.y\n        else:\n            y = log_softmax._log_softmax(x, self.use_cudnn)\n            cupy.exp(y, out=y)\n        gloss = grad_outputs[0]\n        n_unit = t.size // len(t)\n        if self.reduce == 'mean':\n            coeff = gloss * self._coeff\n        else:\n            coeff = gloss[:, None, ...]\n\n        if self.class_weight is None:\n            gx = cuda.elementwise(\n                'T y, S t, T coeff, S n_channel, S n_unit, S ignore_label',\n                'T gx',\n                '''\n                    const int c = (i / n_unit % n_channel);\n                    gx = t == ignore_label ? 0 : coeff * (y - (c == t));\n                ''',\n                'softmax_crossent_bwd')(\n                    y, cupy.expand_dims(t, 1), coeff, x.shape[1],\n                    n_unit, self.ignore_label)\n        else:\n            gx = cuda.elementwise(\n                'T y, raw T w, S t, T coeff, S n_channel, S n_unit, '\n                'S ignore_label',\n                'T gx',\n                '''\n                    const int c = (i / n_unit % n_channel);\n                    gx = t == ignore_label ? 0 : coeff * (y - (c == t)) * w[t];\n                ''',\n                'softmax_crossent_weight_bwd')(\n                    y, self.class_weight, cupy.expand_dims(t, 1), coeff,\n                    x.shape[1], n_unit, self.ignore_label)\n\n        return gx, None\n\n\ndef softmax_cross_entropy(\n        x, t, use_cudnn=True, normalize=True, cache_score=True,\n        class_weight=None, ignore_label=-1, reduce='mean'):\n    \"\"\"Computes cross entropy loss for pre-softmax activations.\n\n    Args:\n        x (~chainer.Variable): Variable holding a multidimensional array whose\n            element indicates unnormalized log probability: the first axis of\n            the variable represents the number of samples, and the second axis\n            represents the number of classes. While this function computes\n            a usual softmax cross entropy if the number of dimensions is equal\n            to 2, it computes a cross entropy of the replicated softmax if the\n            number of dimensions is greater than 2.\n        t (~chainer.Variable): Variable holding an int32 vector of ground truth\n            labels. If ``t[i] == ignore_label``, corresponding ``x[i]`` is\n            ignored.\n        normalize (bool): If ``True``, this function normalizes the cross\n            entropy loss across all instances. If ``False``, it only\n            normalizes along a batch size.\n        cache_score (bool): When it is ``True``, the function stores result\n            of forward computation to use it on backward computation. It\n            reduces computational cost though consumes more memory.\n        class_weight (~numpy.ndarray or ~chainer.cuda.cupy.ndarray): An array\n            that contains constant weights that will be multiplied with the\n            loss values along with the second dimension. The shape of this\n            array should be ``(x.shape[1],)``. If this is not ``None``, each\n            class weight ``class_weight[i]`` is actually multiplied to\n            ``y[:, i]`` that is the corresponding log-softmax output of ``x``\n            and has the same shape as ``x`` before calculating the actual loss\n            value.\n        ignore_label (int): Label value you want to ignore. Its default value\n            is ``-1``. See description of the argument `t`.\n        reduce (str): A string that determines whether to reduce the loss\n            values. If it is ``'mean'``, it computes the sum of the individual\n            cross entropy and normalize it according to ``normalize`` option.\n            If it is ``'no'``, this function computes cross entropy for each\n            instance and does not normalize it (``normalize`` option is\n            ignored). In this case, the loss value of the ignored instance,\n            which has ``ignore_label`` as its target value, is set to ``0``.\n\n    Returns:\n        Variable: A variable holding a scalar array of the cross entropy loss.\n        If ``reduce`` is ``'mean'``, it is a scalar array.\n        If ``reduce`` is ``'no'``, the shape is same as that of ``x``.\n\n    .. note::\n\n       This function is differentiable only by ``x``.\n\n    \"\"\"\n\n    return SoftmaxCrossEntropy(\n        use_cudnn, normalize, cache_score, class_weight, ignore_label, reduce)(\n            x, t)\n", "path": "chainer/functions/loss/softmax_cross_entropy.py"}]}