We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
copy_loss_by_seqlength results in overflow over value of 255 because of sum over boolean tensor
Hi,
The below code results in overflow over value of 255. See related discussion in pytorch [sum of boolean tensor](https://discuss.pytorch.org/t/sum-of-a-boolean-tensor/5719)
https://github.com/OpenNMT/OpenNMT-py/blob/3bd519070ad3b092dc081d2e72d0cd129bd085c7/onmt/modules/CopyGenerator.py#L210-L214

The line 210 should be `tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)`
Thanks.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `onmt/modules/CopyGenerator.py`
Content:
```
1 import torch.nn as nn
2 import torch.nn.functional as F
3 import torch
4 import torch.cuda
5 
6 import onmt
7 import onmt.io
8 from onmt.Utils import aeq
9 
10 
11 class CopyGenerator(nn.Module):
12     """Generator module that additionally considers copying
13     words directly from the source.
14 
15     The main idea is that we have an extended "dynamic dictionary".
16     It contains `|tgt_dict|` words plus an arbitrary number of
17     additional words introduced by the source sentence.
18     For each source sentence we have a `src_map` that maps
19     each source word to an index in `tgt_dict` if it known, or
20     else to an extra word.
21 
22     The copy generator is an extended version of the standard
23     generator that computse three values.
24 
25     * :math:`p_{softmax}` the standard softmax over `tgt_dict`
26     * :math:`p(z)` the probability of instead copying a
27       word from the source, computed using a bernoulli
28     * :math:`p_{copy}` the probility of copying a word instead.
29       taken from the attention distribution directly.
30 
31     The model returns a distribution over the extend dictionary,
32     computed as
33 
34     :math:`p(w) = p(z=1)  p_{copy}(w)  +  p(z=0)  p_{softmax}(w)`
35 
36 
37     .. mermaid::
38 
39        graph BT
40           A[input]
41           S[src_map]
42           B[softmax]
43           BB[switch]
44           C[attn]
45           D[copy]
46           O[output]
47           A --> B
48           A --> BB
49           S --> D
50           C --> D
51           D --> O
52           B --> O
53           BB --> O
54 
55 
56     Args:
57        input_size (int): size of input representation
58        tgt_dict (Vocab): output target dictionary
59 
60     """
61     def __init__(self, input_size, tgt_dict):
62         super(CopyGenerator, self).__init__()
63         self.linear = nn.Linear(input_size, len(tgt_dict))
64         self.linear_copy = nn.Linear(input_size, 1)
65         self.tgt_dict = tgt_dict
66 
67     def forward(self, hidden, attn, src_map):
68         """
69         Compute a distribution over the target dictionary
70         extended by the dynamic dictionary implied by compying
71         source words.
72 
73         Args:
74            hidden (`FloatTensor`): hidden outputs `[batch*tlen, input_size]`
75            attn (`FloatTensor`): attn for each `[batch*tlen, input_size]`
76            src_map (`FloatTensor`):
77              A sparse indicator matrix mapping each source word to
78              its index in the "extended" vocab containing.
79              `[src_len, batch, extra_words]`
80         """
81         # CHECKS
82         batch_by_tlen, _ = hidden.size()
83         batch_by_tlen_, slen = attn.size()
84         slen_, batch, cvocab = src_map.size()
85         aeq(batch_by_tlen, batch_by_tlen_)
86         aeq(slen, slen_)
87 
88         # Original probabilities.
89         logits = self.linear(hidden)
90         logits[:, self.tgt_dict.stoi[onmt.io.PAD_WORD]] = -float('inf')
91         prob = F.softmax(logits)
92 
93         # Probability of copying p(z=1) batch.
94         p_copy = F.sigmoid(self.linear_copy(hidden))
95         # Probibility of not copying: p_{word}(w) * (1 - p(z))
96         out_prob = torch.mul(prob,  1 - p_copy.expand_as(prob))
97         mul_attn = torch.mul(attn, p_copy.expand_as(attn))
98         copy_prob = torch.bmm(mul_attn.view(-1, batch, slen)
99                               .transpose(0, 1),
100                               src_map.transpose(0, 1)).transpose(0, 1)
101         copy_prob = copy_prob.contiguous().view(-1, cvocab)
102         return torch.cat([out_prob, copy_prob], 1)
103 
104 
105 class CopyGeneratorCriterion(object):
106     def __init__(self, vocab_size, force_copy, pad, eps=1e-20):
107         self.force_copy = force_copy
108         self.eps = eps
109         self.offset = vocab_size
110         self.pad = pad
111 
112     def __call__(self, scores, align, target):
113         # Compute unks in align and target for readability
114         align_unk = align.eq(0).float()
115         align_not_unk = align.ne(0).float()
116         target_unk = target.eq(0).float()
117         target_not_unk = target.ne(0).float()
118 
119         # Copy probability of tokens in source
120         out = scores.gather(1, align.view(-1, 1) + self.offset).view(-1)
121         # Set scores for unk to 0 and add eps
122         out = out.mul(align_not_unk) + self.eps
123         # Get scores for tokens in target
124         tmp = scores.gather(1, target.view(-1, 1)).view(-1)
125 
126         # Regular prob (no unks and unks that can't be copied)
127         if not self.force_copy:
128             # Add score for non-unks in target
129             out = out + tmp.mul(target_not_unk)
130             # Add score for when word is unk in both align and tgt
131             out = out + tmp.mul(align_unk).mul(target_unk)
132         else:
133             # Forced copy. Add only probability for not-copied tokens
134             out = out + tmp.mul(align_unk)
135 
136         # Drop padding.
137         loss = -out.log().mul(target.ne(self.pad).float())
138         return loss
139 
140 
141 class CopyGeneratorLossCompute(onmt.Loss.LossComputeBase):
142     """
143     Copy Generator Loss Computation.
144     """
145     def __init__(self, generator, tgt_vocab,
146                  force_copy, normalize_by_length,
147                  eps=1e-20):
148         super(CopyGeneratorLossCompute, self).__init__(
149             generator, tgt_vocab)
150 
151         # We lazily load datasets when there are more than one, so postpone
152         # the setting of cur_dataset.
153         self.cur_dataset = None
154         self.force_copy = force_copy
155         self.normalize_by_length = normalize_by_length
156         self.criterion = CopyGeneratorCriterion(len(tgt_vocab), force_copy,
157                                                 self.padding_idx)
158 
159     def _make_shard_state(self, batch, output, range_, attns):
160         """ See base class for args description. """
161         if getattr(batch, "alignment", None) is None:
162             raise AssertionError("using -copy_attn you need to pass in "
163                                  "-dynamic_dict during preprocess stage.")
164 
165         return {
166             "output": output,
167             "target": batch.tgt[range_[0] + 1: range_[1]],
168             "copy_attn": attns.get("copy"),
169             "align": batch.alignment[range_[0] + 1: range_[1]]
170         }
171 
172     def _compute_loss(self, batch, output, target, copy_attn, align):
173         """
174         Compute the loss. The args must match self._make_shard_state().
175         Args:
176             batch: the current batch.
177             output: the predict output from the model.
178             target: the validate target to compare output with.
179             copy_attn: the copy attention value.
180             align: the align info.
181         """
182         target = target.view(-1)
183         align = align.view(-1)
184         scores = self.generator(self._bottle(output),
185                                 self._bottle(copy_attn),
186                                 batch.src_map)
187         loss = self.criterion(scores, align, target)
188         scores_data = scores.data.clone()
189         scores_data = onmt.io.TextDataset.collapse_copy_scores(
190                 self._unbottle(scores_data, batch.batch_size),
191                 batch, self.tgt_vocab, self.cur_dataset.src_vocabs)
192         scores_data = self._bottle(scores_data)
193 
194         # Correct target copy token instead of <unk>
195         # tgt[i] = align[i] + len(tgt_vocab)
196         # for i such that tgt[i] == 0 and align[i] != 0
197         target_data = target.data.clone()
198         correct_mask = target_data.eq(0) * align.data.ne(0)
199         correct_copy = (align.data + len(self.tgt_vocab)) * correct_mask.long()
200         target_data = target_data + correct_copy
201 
202         # Compute sum of perplexities for stats
203         loss_data = loss.sum().data.clone()
204         stats = self._stats(loss_data, scores_data, target_data)
205 
206         if self.normalize_by_length:
207             # Compute Loss as NLL divided by seq length
208             # Compute Sequence Lengths
209             pad_ix = batch.dataset.fields['tgt'].vocab.stoi[onmt.io.PAD_WORD]
210             tgt_lens = batch.tgt.ne(pad_ix).sum(0).float()
211             # Compute Total Loss per sequence in batch
212             loss = loss.view(-1, batch.batch_size).sum(0)
213             # Divide by length of each sequence and sum
214             loss = torch.div(loss, tgt_lens).sum()
215         else:
216             loss = loss.sum()
217 
218         return loss, stats
219
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/onmt/modules/CopyGenerator.py b/onmt/modules/CopyGenerator.py
--- a/onmt/modules/CopyGenerator.py
+++ b/onmt/modules/CopyGenerator.py
@@ -207,7 +207,7 @@
             # Compute Loss as NLL divided by seq length
             # Compute Sequence Lengths
             pad_ix = batch.dataset.fields['tgt'].vocab.stoi[onmt.io.PAD_WORD]
-            tgt_lens = batch.tgt.ne(pad_ix).sum(0).float()
+            tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)
             # Compute Total Loss per sequence in batch
             loss = loss.view(-1, batch.batch_size).sum(0)
             # Divide by length of each sequence and sum

{"golden_diff": "diff --git a/onmt/modules/CopyGenerator.py b/onmt/modules/CopyGenerator.py\n--- a/onmt/modules/CopyGenerator.py\n+++ b/onmt/modules/CopyGenerator.py\n@@ -207,7 +207,7 @@\n             # Compute Loss as NLL divided by seq length\n             # Compute Sequence Lengths\n             pad_ix = batch.dataset.fields['tgt'].vocab.stoi[onmt.io.PAD_WORD]\n-            tgt_lens = batch.tgt.ne(pad_ix).sum(0).float()\n+            tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)\n             # Compute Total Loss per sequence in batch\n             loss = loss.view(-1, batch.batch_size).sum(0)\n             # Divide by length of each sequence and sum\n", "issue": "copy_loss_by_seqlength results in overflow over value of 255 because of sum over boolean tensor\nHi,\r\nThe below code results in overflow over value of 255. See related discussion in pytorch [sum of boolean tensor](https://discuss.pytorch.org/t/sum-of-a-boolean-tensor/5719)\r\nhttps://github.com/OpenNMT/OpenNMT-py/blob/3bd519070ad3b092dc081d2e72d0cd129bd085c7/onmt/modules/CopyGenerator.py#L210-L214\r\n\r\nThe line 210 should be `tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)`\r\nThanks.\r\n\n", "before_files": [{"content": "import torch.nn as nn\nimport torch.nn.functional as F\nimport torch\nimport torch.cuda\n\nimport onmt\nimport onmt.io\nfrom onmt.Utils import aeq\n\n\nclass CopyGenerator(nn.Module):\n    \"\"\"Generator module that additionally considers copying\n    words directly from the source.\n\n    The main idea is that we have an extended \"dynamic dictionary\".\n    It contains `|tgt_dict|` words plus an arbitrary number of\n    additional words introduced by the source sentence.\n    For each source sentence we have a `src_map` that maps\n    each source word to an index in `tgt_dict` if it known, or\n    else to an extra word.\n\n    The copy generator is an extended version of the standard\n    generator that computse three values.\n\n    * :math:`p_{softmax}` the standard softmax over `tgt_dict`\n    * :math:`p(z)` the probability of instead copying a\n      word from the source, computed using a bernoulli\n    * :math:`p_{copy}` the probility of copying a word instead.\n      taken from the attention distribution directly.\n\n    The model returns a distribution over the extend dictionary,\n    computed as\n\n    :math:`p(w) = p(z=1)  p_{copy}(w)  +  p(z=0)  p_{softmax}(w)`\n\n\n    .. mermaid::\n\n       graph BT\n          A[input]\n          S[src_map]\n          B[softmax]\n          BB[switch]\n          C[attn]\n          D[copy]\n          O[output]\n          A --> B\n          A --> BB\n          S --> D\n          C --> D\n          D --> O\n          B --> O\n          BB --> O\n\n\n    Args:\n       input_size (int): size of input representation\n       tgt_dict (Vocab): output target dictionary\n\n    \"\"\"\n    def __init__(self, input_size, tgt_dict):\n        super(CopyGenerator, self).__init__()\n        self.linear = nn.Linear(input_size, len(tgt_dict))\n        self.linear_copy = nn.Linear(input_size, 1)\n        self.tgt_dict = tgt_dict\n\n    def forward(self, hidden, attn, src_map):\n        \"\"\"\n        Compute a distribution over the target dictionary\n        extended by the dynamic dictionary implied by compying\n        source words.\n\n        Args:\n           hidden (`FloatTensor`): hidden outputs `[batch*tlen, input_size]`\n           attn (`FloatTensor`): attn for each `[batch*tlen, input_size]`\n           src_map (`FloatTensor`):\n             A sparse indicator matrix mapping each source word to\n             its index in the \"extended\" vocab containing.\n             `[src_len, batch, extra_words]`\n        \"\"\"\n        # CHECKS\n        batch_by_tlen, _ = hidden.size()\n        batch_by_tlen_, slen = attn.size()\n        slen_, batch, cvocab = src_map.size()\n        aeq(batch_by_tlen, batch_by_tlen_)\n        aeq(slen, slen_)\n\n        # Original probabilities.\n        logits = self.linear(hidden)\n        logits[:, self.tgt_dict.stoi[onmt.io.PAD_WORD]] = -float('inf')\n        prob = F.softmax(logits)\n\n        # Probability of copying p(z=1) batch.\n        p_copy = F.sigmoid(self.linear_copy(hidden))\n        # Probibility of not copying: p_{word}(w) * (1 - p(z))\n        out_prob = torch.mul(prob,  1 - p_copy.expand_as(prob))\n        mul_attn = torch.mul(attn, p_copy.expand_as(attn))\n        copy_prob = torch.bmm(mul_attn.view(-1, batch, slen)\n                              .transpose(0, 1),\n                              src_map.transpose(0, 1)).transpose(0, 1)\n        copy_prob = copy_prob.contiguous().view(-1, cvocab)\n        return torch.cat([out_prob, copy_prob], 1)\n\n\nclass CopyGeneratorCriterion(object):\n    def __init__(self, vocab_size, force_copy, pad, eps=1e-20):\n        self.force_copy = force_copy\n        self.eps = eps\n        self.offset = vocab_size\n        self.pad = pad\n\n    def __call__(self, scores, align, target):\n        # Compute unks in align and target for readability\n        align_unk = align.eq(0).float()\n        align_not_unk = align.ne(0).float()\n        target_unk = target.eq(0).float()\n        target_not_unk = target.ne(0).float()\n\n        # Copy probability of tokens in source\n        out = scores.gather(1, align.view(-1, 1) + self.offset).view(-1)\n        # Set scores for unk to 0 and add eps\n        out = out.mul(align_not_unk) + self.eps\n        # Get scores for tokens in target\n        tmp = scores.gather(1, target.view(-1, 1)).view(-1)\n\n        # Regular prob (no unks and unks that can't be copied)\n        if not self.force_copy:\n            # Add score for non-unks in target\n            out = out + tmp.mul(target_not_unk)\n            # Add score for when word is unk in both align and tgt\n            out = out + tmp.mul(align_unk).mul(target_unk)\n        else:\n            # Forced copy. Add only probability for not-copied tokens\n            out = out + tmp.mul(align_unk)\n\n        # Drop padding.\n        loss = -out.log().mul(target.ne(self.pad).float())\n        return loss\n\n\nclass CopyGeneratorLossCompute(onmt.Loss.LossComputeBase):\n    \"\"\"\n    Copy Generator Loss Computation.\n    \"\"\"\n    def __init__(self, generator, tgt_vocab,\n                 force_copy, normalize_by_length,\n                 eps=1e-20):\n        super(CopyGeneratorLossCompute, self).__init__(\n            generator, tgt_vocab)\n\n        # We lazily load datasets when there are more than one, so postpone\n        # the setting of cur_dataset.\n        self.cur_dataset = None\n        self.force_copy = force_copy\n        self.normalize_by_length = normalize_by_length\n        self.criterion = CopyGeneratorCriterion(len(tgt_vocab), force_copy,\n                                                self.padding_idx)\n\n    def _make_shard_state(self, batch, output, range_, attns):\n        \"\"\" See base class for args description. \"\"\"\n        if getattr(batch, \"alignment\", None) is None:\n            raise AssertionError(\"using -copy_attn you need to pass in \"\n                                 \"-dynamic_dict during preprocess stage.\")\n\n        return {\n            \"output\": output,\n            \"target\": batch.tgt[range_[0] + 1: range_[1]],\n            \"copy_attn\": attns.get(\"copy\"),\n            \"align\": batch.alignment[range_[0] + 1: range_[1]]\n        }\n\n    def _compute_loss(self, batch, output, target, copy_attn, align):\n        \"\"\"\n        Compute the loss. The args must match self._make_shard_state().\n        Args:\n            batch: the current batch.\n            output: the predict output from the model.\n            target: the validate target to compare output with.\n            copy_attn: the copy attention value.\n            align: the align info.\n        \"\"\"\n        target = target.view(-1)\n        align = align.view(-1)\n        scores = self.generator(self._bottle(output),\n                                self._bottle(copy_attn),\n                                batch.src_map)\n        loss = self.criterion(scores, align, target)\n        scores_data = scores.data.clone()\n        scores_data = onmt.io.TextDataset.collapse_copy_scores(\n                self._unbottle(scores_data, batch.batch_size),\n                batch, self.tgt_vocab, self.cur_dataset.src_vocabs)\n        scores_data = self._bottle(scores_data)\n\n        # Correct target copy token instead of <unk>\n        # tgt[i] = align[i] + len(tgt_vocab)\n        # for i such that tgt[i] == 0 and align[i] != 0\n        target_data = target.data.clone()\n        correct_mask = target_data.eq(0) * align.data.ne(0)\n        correct_copy = (align.data + len(self.tgt_vocab)) * correct_mask.long()\n        target_data = target_data + correct_copy\n\n        # Compute sum of perplexities for stats\n        loss_data = loss.sum().data.clone()\n        stats = self._stats(loss_data, scores_data, target_data)\n\n        if self.normalize_by_length:\n            # Compute Loss as NLL divided by seq length\n            # Compute Sequence Lengths\n            pad_ix = batch.dataset.fields['tgt'].vocab.stoi[onmt.io.PAD_WORD]\n            tgt_lens = batch.tgt.ne(pad_ix).sum(0).float()\n            # Compute Total Loss per sequence in batch\n            loss = loss.view(-1, batch.batch_size).sum(0)\n            # Divide by length of each sequence and sum\n            loss = torch.div(loss, tgt_lens).sum()\n        else:\n            loss = loss.sum()\n\n        return loss, stats\n", "path": "onmt/modules/CopyGenerator.py"}], "after_files": [{"content": "import torch.nn as nn\nimport torch.nn.functional as F\nimport torch\nimport torch.cuda\n\nimport onmt\nimport onmt.io\nfrom onmt.Utils import aeq\n\n\nclass CopyGenerator(nn.Module):\n    \"\"\"Generator module that additionally considers copying\n    words directly from the source.\n\n    The main idea is that we have an extended \"dynamic dictionary\".\n    It contains `|tgt_dict|` words plus an arbitrary number of\n    additional words introduced by the source sentence.\n    For each source sentence we have a `src_map` that maps\n    each source word to an index in `tgt_dict` if it known, or\n    else to an extra word.\n\n    The copy generator is an extended version of the standard\n    generator that computse three values.\n\n    * :math:`p_{softmax}` the standard softmax over `tgt_dict`\n    * :math:`p(z)` the probability of instead copying a\n      word from the source, computed using a bernoulli\n    * :math:`p_{copy}` the probility of copying a word instead.\n      taken from the attention distribution directly.\n\n    The model returns a distribution over the extend dictionary,\n    computed as\n\n    :math:`p(w) = p(z=1)  p_{copy}(w)  +  p(z=0)  p_{softmax}(w)`\n\n\n    .. mermaid::\n\n       graph BT\n          A[input]\n          S[src_map]\n          B[softmax]\n          BB[switch]\n          C[attn]\n          D[copy]\n          O[output]\n          A --> B\n          A --> BB\n          S --> D\n          C --> D\n          D --> O\n          B --> O\n          BB --> O\n\n\n    Args:\n       input_size (int): size of input representation\n       tgt_dict (Vocab): output target dictionary\n\n    \"\"\"\n    def __init__(self, input_size, tgt_dict):\n        super(CopyGenerator, self).__init__()\n        self.linear = nn.Linear(input_size, len(tgt_dict))\n        self.linear_copy = nn.Linear(input_size, 1)\n        self.tgt_dict = tgt_dict\n\n    def forward(self, hidden, attn, src_map):\n        \"\"\"\n        Compute a distribution over the target dictionary\n        extended by the dynamic dictionary implied by compying\n        source words.\n\n        Args:\n           hidden (`FloatTensor`): hidden outputs `[batch*tlen, input_size]`\n           attn (`FloatTensor`): attn for each `[batch*tlen, input_size]`\n           src_map (`FloatTensor`):\n             A sparse indicator matrix mapping each source word to\n             its index in the \"extended\" vocab containing.\n             `[src_len, batch, extra_words]`\n        \"\"\"\n        # CHECKS\n        batch_by_tlen, _ = hidden.size()\n        batch_by_tlen_, slen = attn.size()\n        slen_, batch, cvocab = src_map.size()\n        aeq(batch_by_tlen, batch_by_tlen_)\n        aeq(slen, slen_)\n\n        # Original probabilities.\n        logits = self.linear(hidden)\n        logits[:, self.tgt_dict.stoi[onmt.io.PAD_WORD]] = -float('inf')\n        prob = F.softmax(logits)\n\n        # Probability of copying p(z=1) batch.\n        p_copy = F.sigmoid(self.linear_copy(hidden))\n        # Probibility of not copying: p_{word}(w) * (1 - p(z))\n        out_prob = torch.mul(prob,  1 - p_copy.expand_as(prob))\n        mul_attn = torch.mul(attn, p_copy.expand_as(attn))\n        copy_prob = torch.bmm(mul_attn.view(-1, batch, slen)\n                              .transpose(0, 1),\n                              src_map.transpose(0, 1)).transpose(0, 1)\n        copy_prob = copy_prob.contiguous().view(-1, cvocab)\n        return torch.cat([out_prob, copy_prob], 1)\n\n\nclass CopyGeneratorCriterion(object):\n    def __init__(self, vocab_size, force_copy, pad, eps=1e-20):\n        self.force_copy = force_copy\n        self.eps = eps\n        self.offset = vocab_size\n        self.pad = pad\n\n    def __call__(self, scores, align, target):\n        # Compute unks in align and target for readability\n        align_unk = align.eq(0).float()\n        align_not_unk = align.ne(0).float()\n        target_unk = target.eq(0).float()\n        target_not_unk = target.ne(0).float()\n\n        # Copy probability of tokens in source\n        out = scores.gather(1, align.view(-1, 1) + self.offset).view(-1)\n        # Set scores for unk to 0 and add eps\n        out = out.mul(align_not_unk) + self.eps\n        # Get scores for tokens in target\n        tmp = scores.gather(1, target.view(-1, 1)).view(-1)\n\n        # Regular prob (no unks and unks that can't be copied)\n        if not self.force_copy:\n            # Add score for non-unks in target\n            out = out + tmp.mul(target_not_unk)\n            # Add score for when word is unk in both align and tgt\n            out = out + tmp.mul(align_unk).mul(target_unk)\n        else:\n            # Forced copy. Add only probability for not-copied tokens\n            out = out + tmp.mul(align_unk)\n\n        # Drop padding.\n        loss = -out.log().mul(target.ne(self.pad).float())\n        return loss\n\n\nclass CopyGeneratorLossCompute(onmt.Loss.LossComputeBase):\n    \"\"\"\n    Copy Generator Loss Computation.\n    \"\"\"\n    def __init__(self, generator, tgt_vocab,\n                 force_copy, normalize_by_length,\n                 eps=1e-20):\n        super(CopyGeneratorLossCompute, self).__init__(\n            generator, tgt_vocab)\n\n        # We lazily load datasets when there are more than one, so postpone\n        # the setting of cur_dataset.\n        self.cur_dataset = None\n        self.force_copy = force_copy\n        self.normalize_by_length = normalize_by_length\n        self.criterion = CopyGeneratorCriterion(len(tgt_vocab), force_copy,\n                                                self.padding_idx)\n\n    def _make_shard_state(self, batch, output, range_, attns):\n        \"\"\" See base class for args description. \"\"\"\n        if getattr(batch, \"alignment\", None) is None:\n            raise AssertionError(\"using -copy_attn you need to pass in \"\n                                 \"-dynamic_dict during preprocess stage.\")\n\n        return {\n            \"output\": output,\n            \"target\": batch.tgt[range_[0] + 1: range_[1]],\n            \"copy_attn\": attns.get(\"copy\"),\n            \"align\": batch.alignment[range_[0] + 1: range_[1]]\n        }\n\n    def _compute_loss(self, batch, output, target, copy_attn, align):\n        \"\"\"\n        Compute the loss. The args must match self._make_shard_state().\n        Args:\n            batch: the current batch.\n            output: the predict output from the model.\n            target: the validate target to compare output with.\n            copy_attn: the copy attention value.\n            align: the align info.\n        \"\"\"\n        target = target.view(-1)\n        align = align.view(-1)\n        scores = self.generator(self._bottle(output),\n                                self._bottle(copy_attn),\n                                batch.src_map)\n        loss = self.criterion(scores, align, target)\n        scores_data = scores.data.clone()\n        scores_data = onmt.io.TextDataset.collapse_copy_scores(\n                self._unbottle(scores_data, batch.batch_size),\n                batch, self.tgt_vocab, self.cur_dataset.src_vocabs)\n        scores_data = self._bottle(scores_data)\n\n        # Correct target copy token instead of <unk>\n        # tgt[i] = align[i] + len(tgt_vocab)\n        # for i such that tgt[i] == 0 and align[i] != 0\n        target_data = target.data.clone()\n        correct_mask = target_data.eq(0) * align.data.ne(0)\n        correct_copy = (align.data + len(self.tgt_vocab)) * correct_mask.long()\n        target_data = target_data + correct_copy\n\n        # Compute sum of perplexities for stats\n        loss_data = loss.sum().data.clone()\n        stats = self._stats(loss_data, scores_data, target_data)\n\n        if self.normalize_by_length:\n            # Compute Loss as NLL divided by seq length\n            # Compute Sequence Lengths\n            pad_ix = batch.dataset.fields['tgt'].vocab.stoi[onmt.io.PAD_WORD]\n            tgt_lens = batch.tgt.ne(pad_ix).float().sum(0)\n            # Compute Total Loss per sequence in batch\n            loss = loss.view(-1, batch.batch_size).sum(0)\n            # Divide by length of each sequence and sum\n            loss = torch.div(loss, tgt_lens).sum()\n        else:\n            loss = loss.sum()\n\n        return loss, stats\n", "path": "onmt/modules/CopyGenerator.py"}]}