prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Source/Target length distinction
  ## Preprocess parameters
  Removed parameter `-seq_length`
  New parameters `-src_seq_length` and `-tgt_seq_length`
  
  ---
  
  ## Training speed token/s
  In both LUA/PyTorch OpenNMT, the training process prints a speed, in token/sec, but:
  
  * LUA OpenNMT is printing source token/sec
  * PyOpenNMT is printing target token/sec
  
  This can lead to important differences, especially when src/tgt sequence length are different (e.g. summarization), and therefore lead to false conclusion about performances.
  
  See also: [pytoch/example/issue#75](https://github.com/pytorch/examples/issues/75)
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `onmt/inputters/dataset_base.py`
  Content:
  ```
  1 # coding: utf-8
  2 
  3 from itertools import chain
  4 from collections import Counter
  5 import codecs
  6 
  7 import torch
  8 from torchtext.data import Example, Dataset
  9 from torchtext.vocab import Vocab
  10 
  11 
  12 class DatasetBase(Dataset):
  13     """
  14     A dataset is an object that accepts sequences of raw data (sentence pairs
  15     in the case of machine translation) and fields which describe how this
  16     raw data should be processed to produce tensors. When a dataset is
  17     instantiated, it applies the fields' preprocessing pipeline (but not
  18     the bit that numericalizes it or turns it into batch tensors) to the raw
  19     data, producing a list of torchtext.data.Example objects. torchtext's
  20     iterators then know how to use these examples to make batches.
  21 
  22     Datasets in OpenNMT take three positional arguments:
  23 
  24     `fields`: a dict with the structure returned by inputters.get_fields().
  25         keys match the keys of items yielded by the src_examples_iter or
  26         tgt_examples_iter, while values are lists of (name, Field) pairs.
  27         An attribute with this name will be created for each Example object,
  28         and its value will be the result of applying the Field to the data
  29         that matches the key. The advantage of having sequences of fields
  30         for each piece of raw input is that it allows for the dataset to store
  31         multiple `views` of each input, which allows for easy implementation
  32         of token-level features, mixed word- and character-level models, and
  33         so on.
  34     `src_examples_iter`: a sequence of dicts. Each dict's keys should be a
  35         subset of the keys in `fields`.
  36     `tgt_examples_iter`: like `src_examples_iter`, but may be None (this is
  37         the case at translation time if no target is specified).
  38 
  39     `filter_pred` if specified, a function that accepts Example objects and
  40         returns a boolean value indicating whether to include that example
  41         in the dataset.
  42 
  43     The resulting dataset will have three attributes (todo: also src_vocabs):
  44 
  45      `examples`: a list of `torchtext.data.Example` objects with attributes as
  46         described above.
  47      `fields`: a dictionary whose keys are strings with the same names as the
  48         attributes of the elements of `examples` and whose values are
  49         the corresponding `torchtext.data.Field` objects. NOTE: this is not
  50         the same structure as in the fields argument passed to the constructor.
  51     """
  52 
  53     def __getstate__(self):
  54         return self.__dict__
  55 
  56     def __setstate__(self, _d):
  57         self.__dict__.update(_d)
  58 
  59     def __reduce_ex__(self, proto):
  60         # This is a hack. Something is broken with torch pickle.
  61         return super(DatasetBase, self).__reduce_ex__()
  62 
  63     def __init__(self, fields, src_examples_iter, tgt_examples_iter,
  64                  filter_pred=None):
  65 
  66         dynamic_dict = 'src_map' in fields and 'alignment' in fields
  67 
  68         if tgt_examples_iter is not None:
  69             examples_iter = (self._join_dicts(src, tgt) for src, tgt in
  70                              zip(src_examples_iter, tgt_examples_iter))
  71         else:
  72             examples_iter = src_examples_iter
  73 
  74         # self.src_vocabs is used in collapse_copy_scores and Translator.py
  75         self.src_vocabs = []
  76         examples = []
  77         for ex_dict in examples_iter:
  78             if dynamic_dict:
  79                 src_field = fields['src'][0][1]
  80                 tgt_field = fields['tgt'][0][1]
  81                 src_vocab, ex_dict = self._dynamic_dict(
  82                     ex_dict, src_field, tgt_field)
  83                 self.src_vocabs.append(src_vocab)
  84             ex_fields = {k: v for k, v in fields.items() if k in ex_dict}
  85             ex = Example.fromdict(ex_dict, ex_fields)
  86             examples.append(ex)
  87 
  88         # the dataset's self.fields should have the same attributes as examples
  89         fields = dict(chain.from_iterable(ex_fields.values()))
  90 
  91         super(DatasetBase, self).__init__(examples, fields, filter_pred)
  92 
  93     def save(self, path, remove_fields=True):
  94         if remove_fields:
  95             self.fields = []
  96         torch.save(self, path)
  97 
  98     def _join_dicts(self, *args):
  99         """
  100         Args:
  101             dictionaries with disjoint keys.
  102 
  103         Returns:
  104             a single dictionary that has the union of these keys.
  105         """
  106         return dict(chain(*[d.items() for d in args]))
  107 
  108     def _dynamic_dict(self, example, src_field, tgt_field):
  109         src = src_field.tokenize(example["src"])
  110         # make a small vocab containing just the tokens in the source sequence
  111         unk = src_field.unk_token
  112         pad = src_field.pad_token
  113         src_vocab = Vocab(Counter(src), specials=[unk, pad])
  114         # Map source tokens to indices in the dynamic dict.
  115         src_map = torch.LongTensor([src_vocab.stoi[w] for w in src])
  116         example["src_map"] = src_map
  117 
  118         if "tgt" in example:
  119             tgt = tgt_field.tokenize(example["tgt"])
  120             mask = torch.LongTensor(
  121                 [0] + [src_vocab.stoi[w] for w in tgt] + [0])
  122             example["alignment"] = mask
  123         return src_vocab, example
  124 
  125     @property
  126     def can_copy(self):
  127         return False
  128 
  129     @classmethod
  130     def _read_file(cls, path):
  131         with codecs.open(path, "r", "utf-8") as f:
  132             for line in f:
  133                 yield line
  134
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/onmt/inputters/dataset_base.py b/onmt/inputters/dataset_base.py
  --- a/onmt/inputters/dataset_base.py
  +++ b/onmt/inputters/dataset_base.py
  @@ -50,16 +50,6 @@
           the same structure as in the fields argument passed to the constructor.
       """
   
  -    def __getstate__(self):
  -        return self.__dict__
  -
  -    def __setstate__(self, _d):
  -        self.__dict__.update(_d)
  -
  -    def __reduce_ex__(self, proto):
  -        # This is a hack. Something is broken with torch pickle.
  -        return super(DatasetBase, self).__reduce_ex__()
  -
       def __init__(self, fields, src_examples_iter, tgt_examples_iter,
                    filter_pred=None):
   
  @@ -90,6 +80,15 @@
   
           super(DatasetBase, self).__init__(examples, fields, filter_pred)
   
  +    def __getattr__(self, attr):
  +        # avoid infinite recursion when fields isn't defined
  +        if 'fields' not in vars(self):
  +            raise AttributeError
  +        if attr in self.fields:
  +            return (getattr(x, attr) for x in self.examples)
  +        else:
  +            raise AttributeError
  +
       def save(self, path, remove_fields=True):
           if remove_fields:
               self.fields = []
  
verification_info: {"golden_diff": "diff --git a/onmt/inputters/dataset_base.py b/onmt/inputters/dataset_base.py\n--- a/onmt/inputters/dataset_base.py\n+++ b/onmt/inputters/dataset_base.py\n@@ -50,16 +50,6 @@\n         the same structure as in the fields argument passed to the constructor.\n     \"\"\"\n \n-    def __getstate__(self):\n-        return self.__dict__\n-\n-    def __setstate__(self, _d):\n-        self.__dict__.update(_d)\n-\n-    def __reduce_ex__(self, proto):\n-        # This is a hack. Something is broken with torch pickle.\n-        return super(DatasetBase, self).__reduce_ex__()\n-\n     def __init__(self, fields, src_examples_iter, tgt_examples_iter,\n                  filter_pred=None):\n \n@@ -90,6 +80,15 @@\n \n         super(DatasetBase, self).__init__(examples, fields, filter_pred)\n \n+    def __getattr__(self, attr):\n+        # avoid infinite recursion when fields isn't defined\n+        if 'fields' not in vars(self):\n+            raise AttributeError\n+        if attr in self.fields:\n+            return (getattr(x, attr) for x in self.examples)\n+        else:\n+            raise AttributeError\n+\n     def save(self, path, remove_fields=True):\n         if remove_fields:\n             self.fields = []\n", "issue": "Source/Target length distinction\n## Preprocess parameters\r\nRemoved parameter `-seq_length`\r\nNew parameters `-src_seq_length` and `-tgt_seq_length`\r\n\r\n---\r\n\r\n## Training speed token/s\r\nIn both LUA/PyTorch OpenNMT, the training process prints a speed, in token/sec, but:\r\n\r\n* LUA OpenNMT is printing source token/sec\r\n* PyOpenNMT is printing target token/sec\r\n\r\nThis can lead to important differences, especially when src/tgt sequence length are different (e.g. summarization), and therefore lead to false conclusion about performances.\r\n\r\nSee also: [pytoch/example/issue#75](https://github.com/pytorch/examples/issues/75)\n", "before_files": [{"content": "# coding: utf-8\n\nfrom itertools import chain\nfrom collections import Counter\nimport codecs\n\nimport torch\nfrom torchtext.data import Example, Dataset\nfrom torchtext.vocab import Vocab\n\n\nclass DatasetBase(Dataset):\n    \"\"\"\n    A dataset is an object that accepts sequences of raw data (sentence pairs\n    in the case of machine translation) and fields which describe how this\n    raw data should be processed to produce tensors. When a dataset is\n    instantiated, it applies the fields' preprocessing pipeline (but not\n    the bit that numericalizes it or turns it into batch tensors) to the raw\n    data, producing a list of torchtext.data.Example objects. torchtext's\n    iterators then know how to use these examples to make batches.\n\n    Datasets in OpenNMT take three positional arguments:\n\n    `fields`: a dict with the structure returned by inputters.get_fields().\n        keys match the keys of items yielded by the src_examples_iter or\n        tgt_examples_iter, while values are lists of (name, Field) pairs.\n        An attribute with this name will be created for each Example object,\n        and its value will be the result of applying the Field to the data\n        that matches the key. The advantage of having sequences of fields\n        for each piece of raw input is that it allows for the dataset to store\n        multiple `views` of each input, which allows for easy implementation\n        of token-level features, mixed word- and character-level models, and\n        so on.\n    `src_examples_iter`: a sequence of dicts. Each dict's keys should be a\n        subset of the keys in `fields`.\n    `tgt_examples_iter`: like `src_examples_iter`, but may be None (this is\n        the case at translation time if no target is specified).\n\n    `filter_pred` if specified, a function that accepts Example objects and\n        returns a boolean value indicating whether to include that example\n        in the dataset.\n\n    The resulting dataset will have three attributes (todo: also src_vocabs):\n\n     `examples`: a list of `torchtext.data.Example` objects with attributes as\n        described above.\n     `fields`: a dictionary whose keys are strings with the same names as the\n        attributes of the elements of `examples` and whose values are\n        the corresponding `torchtext.data.Field` objects. NOTE: this is not\n        the same structure as in the fields argument passed to the constructor.\n    \"\"\"\n\n    def __getstate__(self):\n        return self.__dict__\n\n    def __setstate__(self, _d):\n        self.__dict__.update(_d)\n\n    def __reduce_ex__(self, proto):\n        # This is a hack. Something is broken with torch pickle.\n        return super(DatasetBase, self).__reduce_ex__()\n\n    def __init__(self, fields, src_examples_iter, tgt_examples_iter,\n                 filter_pred=None):\n\n        dynamic_dict = 'src_map' in fields and 'alignment' in fields\n\n        if tgt_examples_iter is not None:\n            examples_iter = (self._join_dicts(src, tgt) for src, tgt in\n                             zip(src_examples_iter, tgt_examples_iter))\n        else:\n            examples_iter = src_examples_iter\n\n        # self.src_vocabs is used in collapse_copy_scores and Translator.py\n        self.src_vocabs = []\n        examples = []\n        for ex_dict in examples_iter:\n            if dynamic_dict:\n                src_field = fields['src'][0][1]\n                tgt_field = fields['tgt'][0][1]\n                src_vocab, ex_dict = self._dynamic_dict(\n                    ex_dict, src_field, tgt_field)\n                self.src_vocabs.append(src_vocab)\n            ex_fields = {k: v for k, v in fields.items() if k in ex_dict}\n            ex = Example.fromdict(ex_dict, ex_fields)\n            examples.append(ex)\n\n        # the dataset's self.fields should have the same attributes as examples\n        fields = dict(chain.from_iterable(ex_fields.values()))\n\n        super(DatasetBase, self).__init__(examples, fields, filter_pred)\n\n    def save(self, path, remove_fields=True):\n        if remove_fields:\n            self.fields = []\n        torch.save(self, path)\n\n    def _join_dicts(self, *args):\n        \"\"\"\n        Args:\n            dictionaries with disjoint keys.\n\n        Returns:\n            a single dictionary that has the union of these keys.\n        \"\"\"\n        return dict(chain(*[d.items() for d in args]))\n\n    def _dynamic_dict(self, example, src_field, tgt_field):\n        src = src_field.tokenize(example[\"src\"])\n        # make a small vocab containing just the tokens in the source sequence\n        unk = src_field.unk_token\n        pad = src_field.pad_token\n        src_vocab = Vocab(Counter(src), specials=[unk, pad])\n        # Map source tokens to indices in the dynamic dict.\n        src_map = torch.LongTensor([src_vocab.stoi[w] for w in src])\n        example[\"src_map\"] = src_map\n\n        if \"tgt\" in example:\n            tgt = tgt_field.tokenize(example[\"tgt\"])\n            mask = torch.LongTensor(\n                [0] + [src_vocab.stoi[w] for w in tgt] + [0])\n            example[\"alignment\"] = mask\n        return src_vocab, example\n\n    @property\n    def can_copy(self):\n        return False\n\n    @classmethod\n    def _read_file(cls, path):\n        with codecs.open(path, \"r\", \"utf-8\") as f:\n            for line in f:\n                yield line\n", "path": "onmt/inputters/dataset_base.py"}], "after_files": [{"content": "# coding: utf-8\n\nfrom itertools import chain\nfrom collections import Counter\nimport codecs\n\nimport torch\nfrom torchtext.data import Example, Dataset\nfrom torchtext.vocab import Vocab\n\n\nclass DatasetBase(Dataset):\n    \"\"\"\n    A dataset is an object that accepts sequences of raw data (sentence pairs\n    in the case of machine translation) and fields which describe how this\n    raw data should be processed to produce tensors. When a dataset is\n    instantiated, it applies the fields' preprocessing pipeline (but not\n    the bit that numericalizes it or turns it into batch tensors) to the raw\n    data, producing a list of torchtext.data.Example objects. torchtext's\n    iterators then know how to use these examples to make batches.\n\n    Datasets in OpenNMT take three positional arguments:\n\n    `fields`: a dict with the structure returned by inputters.get_fields().\n        keys match the keys of items yielded by the src_examples_iter or\n        tgt_examples_iter, while values are lists of (name, Field) pairs.\n        An attribute with this name will be created for each Example object,\n        and its value will be the result of applying the Field to the data\n        that matches the key. The advantage of having sequences of fields\n        for each piece of raw input is that it allows for the dataset to store\n        multiple `views` of each input, which allows for easy implementation\n        of token-level features, mixed word- and character-level models, and\n        so on.\n    `src_examples_iter`: a sequence of dicts. Each dict's keys should be a\n        subset of the keys in `fields`.\n    `tgt_examples_iter`: like `src_examples_iter`, but may be None (this is\n        the case at translation time if no target is specified).\n\n    `filter_pred` if specified, a function that accepts Example objects and\n        returns a boolean value indicating whether to include that example\n        in the dataset.\n\n    The resulting dataset will have three attributes (todo: also src_vocabs):\n\n     `examples`: a list of `torchtext.data.Example` objects with attributes as\n        described above.\n     `fields`: a dictionary whose keys are strings with the same names as the\n        attributes of the elements of `examples` and whose values are\n        the corresponding `torchtext.data.Field` objects. NOTE: this is not\n        the same structure as in the fields argument passed to the constructor.\n    \"\"\"\n\n    def __init__(self, fields, src_examples_iter, tgt_examples_iter,\n                 filter_pred=None):\n\n        dynamic_dict = 'src_map' in fields and 'alignment' in fields\n\n        if tgt_examples_iter is not None:\n            examples_iter = (self._join_dicts(src, tgt) for src, tgt in\n                             zip(src_examples_iter, tgt_examples_iter))\n        else:\n            examples_iter = src_examples_iter\n\n        # self.src_vocabs is used in collapse_copy_scores and Translator.py\n        self.src_vocabs = []\n        examples = []\n        for ex_dict in examples_iter:\n            if dynamic_dict:\n                src_field = fields['src'][0][1]\n                tgt_field = fields['tgt'][0][1]\n                src_vocab, ex_dict = self._dynamic_dict(\n                    ex_dict, src_field, tgt_field)\n                self.src_vocabs.append(src_vocab)\n            ex_fields = {k: v for k, v in fields.items() if k in ex_dict}\n            ex = Example.fromdict(ex_dict, ex_fields)\n            examples.append(ex)\n\n        # the dataset's self.fields should have the same attributes as examples\n        fields = dict(chain.from_iterable(ex_fields.values()))\n\n        super(DatasetBase, self).__init__(examples, fields, filter_pred)\n\n    def __getattr__(self, attr):\n        # avoid infinite recursion when fields isn't defined\n        if 'fields' not in vars(self):\n            raise AttributeError\n        if attr in self.fields:\n            return (getattr(x, attr) for x in self.examples)\n        else:\n            raise AttributeError\n\n    def save(self, path, remove_fields=True):\n        if remove_fields:\n            self.fields = []\n        torch.save(self, path)\n\n    def _join_dicts(self, *args):\n        \"\"\"\n        Args:\n            dictionaries with disjoint keys.\n\n        Returns:\n            a single dictionary that has the union of these keys.\n        \"\"\"\n        return dict(chain(*[d.items() for d in args]))\n\n    def _dynamic_dict(self, example, src_field, tgt_field):\n        src = src_field.tokenize(example[\"src\"])\n        # make a small vocab containing just the tokens in the source sequence\n        unk = src_field.unk_token\n        pad = src_field.pad_token\n        src_vocab = Vocab(Counter(src), specials=[unk, pad])\n        # Map source tokens to indices in the dynamic dict.\n        src_map = torch.LongTensor([src_vocab.stoi[w] for w in src])\n        example[\"src_map\"] = src_map\n\n        if \"tgt\" in example:\n            tgt = tgt_field.tokenize(example[\"tgt\"])\n            mask = torch.LongTensor(\n                [0] + [src_vocab.stoi[w] for w in tgt] + [0])\n            example[\"alignment\"] = mask\n        return src_vocab, example\n\n    @property\n    def can_copy(self):\n        return False\n\n    @classmethod\n    def _read_file(cls, path):\n        with codecs.open(path, \"r\", \"utf-8\") as f:\n            for line in f:\n                yield line\n", "path": "onmt/inputters/dataset_base.py"}]}