prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Building Dict is too slow
  **Bug description**
  ```
  from parlai.scripts.train_model import TrainModel
  
  TrainModel.main(
      # similar to before
      task='amazon_qa',
      model='projects.wizard_of_wikipedia.generator.agents:EndToEndAgent',
      model_file='/tmp/end2end_generator/model',
  
      # initialize with a pretrained model
      init_model='zoo:wizard_of_wikipedia/end2end_generator/model',
  
      # arguments we get from the pretrained model.
      # Unfortunately, these must be looked up separately for each model.
      # eps
      dict_lower=True,
      dict_tokenizer='bpe',
      n_layers=5,
      n_heads=2,
      dropout=0.20,
      ffn_size=512,
      embedding_size=256,
      log_every_n_secs=10,
      validation_patience=12,
      validation_metric='ppl',
      validation_metric_mode='min',
      validation_every_n_epochs=0.5,
      n_positions=128,
      truncate=128,
      max_knowledge=32,
      knowledge_alpha=0.95,
      knowledge_truncate=32,
      learningrate=5e-4,
      warmup_updates=5000,
      clip=0.1,
      lr_scheduler='invsqrt',
      embedding_type='fasttext',
      beam_size=1,
      skip_generation=False,
      batchsize=64,
  )
  
  ```
  I am trying to train amazon_qa task on wizard of Wikipedia model, just to experiment it out, I am not sure if it will work but when I run this script it says creating a task and goes to next stage building dictionary it just becomes to slow of a process.
  
  <img width="1021" alt="Screenshot 2020-07-18 at 10 42 32 PM" src="https://user-images.githubusercontent.com/45225143/87858114-2f34b380-c949-11ea-9928-3bfc77fa91c8.png">
  
  Like has been around 2 hrs and it hasn't crossed 0% yet.
  Can anyone please point me out the error, thanks.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `parlai/tasks/amazon_qa/agents.py`
  Content:
  ```
  1 #!/usr/bin/env python3
  2 
  3 # Copyright (c) Facebook, Inc. and its affiliates.
  4 # This source code is licensed under the MIT license found in the
  5 # LICENSE file in the root directory of this source tree.
  6 
  7 from parlai.core.teachers import FixedDialogTeacher
  8 from .build import build, RESOURCES
  9 import os
  10 import json
  11 
  12 
  13 class DefaultTeacher(FixedDialogTeacher):
  14     def __init__(self, opt, shared=None):
  15         # store datatype
  16         super().__init__(opt, shared)
  17 
  18         dt = opt['datatype'].split(':')[0]
  19         if dt != 'train':
  20             raise RuntimeError('Not valid datatype (only train).')
  21 
  22         if shared:
  23             self.data = shared['data']
  24         else:
  25             build(opt)
  26             self._setup_data()
  27         self.reset()
  28 
  29     def num_episodes(self):
  30         return len(self.data)
  31 
  32     def num_examples(self):
  33         return sum([len(x) for x in self.data])
  34 
  35     def _setup_data(self):
  36         self.existing_keys = [
  37             'question',
  38             'answer',
  39             'asin',
  40             'questionType',
  41             'questionTime',
  42             'askerID',
  43             'answerType',
  44             'answerTime',
  45             'unixTime',
  46             'answererID',
  47             'helpful',
  48             'answerScore',
  49         ]
  50 
  51         self.data = []
  52 
  53         def create_entry_single(episode):
  54             entry = []
  55             for key in self.existing_keys:
  56                 if key in episode:
  57                     entry.append(str(episode[key]))
  58                 else:
  59                     entry.append('N/A')
  60             return entry
  61 
  62         def create_entry_multiple(episode):
  63             entries = []
  64 
  65             for question in episode['questions']:
  66                 new_episode = dict()
  67                 new_episode['asin'] = episode['asin']
  68                 new_episode['askerID'] = question['askerID']
  69                 new_episode['questionTime'] = question['questionTime']
  70                 new_episode['quesitonType'] = question['questionType']
  71                 new_episode['question'] = question['questionText']
  72 
  73                 for answer in question['answers']:
  74                     answer.update(new_episode)
  75                     answer['answer'] = answer['answerText']
  76                     entries.append([create_entry_single(answer)])
  77 
  78             return entries
  79 
  80         fpath = os.path.join(self.opt['datapath'], 'AmazonQA')
  81         for i, f in enumerate(RESOURCES):
  82             json_file = f.file_name[:-3]
  83             file_path = os.path.join(fpath, json_file)
  84 
  85             with open(file_path, 'r') as infile:
  86                 data = infile.read()
  87                 new_data = data.replace('}\n{', '},\n{')
  88                 json_data = json.loads(f'[{new_data}]')
  89 
  90             for ep in json_data:
  91                 # First 20 datasets have a different format than those later
  92                 if i < 21:
  93                     self.data.append([create_entry_single(ep)])
  94                 else:
  95                     self.data += create_entry_multiple(ep)
  96 
  97     def get(self, episode_idx, entry_idx=0):
  98         ep = self.data[episode_idx]
  99         entry = ep[entry_idx]
  100         action = dict()
  101         action['id'] = episode_idx
  102         for i, key in enumerate(self.existing_keys):
  103             if i < 2:
  104                 continue
  105             action[key] = entry[i]
  106         action['episode_done'] = True
  107         action['text'] = entry[0]
  108         action['labels'] = [entry[1]]
  109 
  110         return action
  111 
  112     def share(self):
  113         shared = super().share()
  114         shared['data'] = self.data
  115         return shared
  116
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/parlai/tasks/amazon_qa/agents.py b/parlai/tasks/amazon_qa/agents.py
  --- a/parlai/tasks/amazon_qa/agents.py
  +++ b/parlai/tasks/amazon_qa/agents.py
  @@ -10,6 +10,22 @@
   import json
   
   
  +EXISTING_KEYS = [
  +    'question',
  +    'answer',
  +    'asin',
  +    'questionType',
  +    'questionTime',
  +    'askerID',
  +    'answerType',
  +    'answerTime',
  +    'unixTime',
  +    'answererID',
  +    'helpful',
  +    'answerScore',
  +]
  +
  +
   class DefaultTeacher(FixedDialogTeacher):
       def __init__(self, opt, shared=None):
           # store datatype
  @@ -21,38 +37,27 @@
   
           if shared:
               self.data = shared['data']
  +            self.num_ex = shared['num_ex']
  +            self.num_ep = shared['num_ep']
           else:
               build(opt)
               self._setup_data()
  +            self.num_ex = sum([len(x) for x in self.data])
  +            self.num_ep = len(self.data)
           self.reset()
   
       def num_episodes(self):
  -        return len(self.data)
  +        return self.num_ep
   
       def num_examples(self):
  -        return sum([len(x) for x in self.data])
  +        return self.num_ex
   
       def _setup_data(self):
  -        self.existing_keys = [
  -            'question',
  -            'answer',
  -            'asin',
  -            'questionType',
  -            'questionTime',
  -            'askerID',
  -            'answerType',
  -            'answerTime',
  -            'unixTime',
  -            'answererID',
  -            'helpful',
  -            'answerScore',
  -        ]
  -
           self.data = []
   
           def create_entry_single(episode):
               entry = []
  -            for key in self.existing_keys:
  +            for key in EXISTING_KEYS:
                   if key in episode:
                       entry.append(str(episode[key]))
                   else:
  @@ -99,7 +104,7 @@
           entry = ep[entry_idx]
           action = dict()
           action['id'] = episode_idx
  -        for i, key in enumerate(self.existing_keys):
  +        for i, key in enumerate(EXISTING_KEYS):
               if i < 2:
                   continue
               action[key] = entry[i]
  @@ -112,4 +117,6 @@
       def share(self):
           shared = super().share()
           shared['data'] = self.data
  +        shared['num_ex'] = self.num_ex
  +        shared['num_ep'] = self.num_ep
           return shared
  
verification_info: {"golden_diff": "diff --git a/parlai/tasks/amazon_qa/agents.py b/parlai/tasks/amazon_qa/agents.py\n--- a/parlai/tasks/amazon_qa/agents.py\n+++ b/parlai/tasks/amazon_qa/agents.py\n@@ -10,6 +10,22 @@\n import json\n \n \n+EXISTING_KEYS = [\n+    'question',\n+    'answer',\n+    'asin',\n+    'questionType',\n+    'questionTime',\n+    'askerID',\n+    'answerType',\n+    'answerTime',\n+    'unixTime',\n+    'answererID',\n+    'helpful',\n+    'answerScore',\n+]\n+\n+\n class DefaultTeacher(FixedDialogTeacher):\n     def __init__(self, opt, shared=None):\n         # store datatype\n@@ -21,38 +37,27 @@\n \n         if shared:\n             self.data = shared['data']\n+            self.num_ex = shared['num_ex']\n+            self.num_ep = shared['num_ep']\n         else:\n             build(opt)\n             self._setup_data()\n+            self.num_ex = sum([len(x) for x in self.data])\n+            self.num_ep = len(self.data)\n         self.reset()\n \n     def num_episodes(self):\n-        return len(self.data)\n+        return self.num_ep\n \n     def num_examples(self):\n-        return sum([len(x) for x in self.data])\n+        return self.num_ex\n \n     def _setup_data(self):\n-        self.existing_keys = [\n-            'question',\n-            'answer',\n-            'asin',\n-            'questionType',\n-            'questionTime',\n-            'askerID',\n-            'answerType',\n-            'answerTime',\n-            'unixTime',\n-            'answererID',\n-            'helpful',\n-            'answerScore',\n-        ]\n-\n         self.data = []\n \n         def create_entry_single(episode):\n             entry = []\n-            for key in self.existing_keys:\n+            for key in EXISTING_KEYS:\n                 if key in episode:\n                     entry.append(str(episode[key]))\n                 else:\n@@ -99,7 +104,7 @@\n         entry = ep[entry_idx]\n         action = dict()\n         action['id'] = episode_idx\n-        for i, key in enumerate(self.existing_keys):\n+        for i, key in enumerate(EXISTING_KEYS):\n             if i < 2:\n                 continue\n             action[key] = entry[i]\n@@ -112,4 +117,6 @@\n     def share(self):\n         shared = super().share()\n         shared['data'] = self.data\n+        shared['num_ex'] = self.num_ex\n+        shared['num_ep'] = self.num_ep\n         return shared\n", "issue": "Building Dict is too slow\n**Bug description**\r\n```\r\nfrom parlai.scripts.train_model import TrainModel\r\n\r\nTrainModel.main(\r\n    # similar to before\r\n    task='amazon_qa',\r\n    model='projects.wizard_of_wikipedia.generator.agents:EndToEndAgent',\r\n    model_file='/tmp/end2end_generator/model',\r\n\r\n    # initialize with a pretrained model\r\n    init_model='zoo:wizard_of_wikipedia/end2end_generator/model',\r\n\r\n    # arguments we get from the pretrained model.\r\n    # Unfortunately, these must be looked up separately for each model.\r\n    # eps\r\n    dict_lower=True,\r\n    dict_tokenizer='bpe',\r\n    n_layers=5,\r\n    n_heads=2,\r\n    dropout=0.20,\r\n    ffn_size=512,\r\n    embedding_size=256,\r\n    log_every_n_secs=10,\r\n    validation_patience=12,\r\n    validation_metric='ppl',\r\n    validation_metric_mode='min',\r\n    validation_every_n_epochs=0.5,\r\n    n_positions=128,\r\n    truncate=128,\r\n    max_knowledge=32,\r\n    knowledge_alpha=0.95,\r\n    knowledge_truncate=32,\r\n    learningrate=5e-4,\r\n    warmup_updates=5000,\r\n    clip=0.1,\r\n    lr_scheduler='invsqrt',\r\n    embedding_type='fasttext',\r\n    beam_size=1,\r\n    skip_generation=False,\r\n    batchsize=64,\r\n)\r\n\r\n```\r\nI am trying to train amazon_qa task on wizard of Wikipedia model, just to experiment it out, I am not sure if it will work but when I run this script it says creating a task and goes to next stage building dictionary it just becomes to slow of a process.\r\n\r\n<img width=\"1021\" alt=\"Screenshot 2020-07-18 at 10 42 32 PM\" src=\"https://user-images.githubusercontent.com/45225143/87858114-2f34b380-c949-11ea-9928-3bfc77fa91c8.png\">\r\n\r\nLike has been around 2 hrs and it hasn't crossed 0% yet.\r\nCan anyone please point me out the error, thanks.\r\n\n", "before_files": [{"content": "#!/usr/bin/env python3\n\n# Copyright (c) Facebook, Inc. and its affiliates.\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom parlai.core.teachers import FixedDialogTeacher\nfrom .build import build, RESOURCES\nimport os\nimport json\n\n\nclass DefaultTeacher(FixedDialogTeacher):\n    def __init__(self, opt, shared=None):\n        # store datatype\n        super().__init__(opt, shared)\n\n        dt = opt['datatype'].split(':')[0]\n        if dt != 'train':\n            raise RuntimeError('Not valid datatype (only train).')\n\n        if shared:\n            self.data = shared['data']\n        else:\n            build(opt)\n            self._setup_data()\n        self.reset()\n\n    def num_episodes(self):\n        return len(self.data)\n\n    def num_examples(self):\n        return sum([len(x) for x in self.data])\n\n    def _setup_data(self):\n        self.existing_keys = [\n            'question',\n            'answer',\n            'asin',\n            'questionType',\n            'questionTime',\n            'askerID',\n            'answerType',\n            'answerTime',\n            'unixTime',\n            'answererID',\n            'helpful',\n            'answerScore',\n        ]\n\n        self.data = []\n\n        def create_entry_single(episode):\n            entry = []\n            for key in self.existing_keys:\n                if key in episode:\n                    entry.append(str(episode[key]))\n                else:\n                    entry.append('N/A')\n            return entry\n\n        def create_entry_multiple(episode):\n            entries = []\n\n            for question in episode['questions']:\n                new_episode = dict()\n                new_episode['asin'] = episode['asin']\n                new_episode['askerID'] = question['askerID']\n                new_episode['questionTime'] = question['questionTime']\n                new_episode['quesitonType'] = question['questionType']\n                new_episode['question'] = question['questionText']\n\n                for answer in question['answers']:\n                    answer.update(new_episode)\n                    answer['answer'] = answer['answerText']\n                    entries.append([create_entry_single(answer)])\n\n            return entries\n\n        fpath = os.path.join(self.opt['datapath'], 'AmazonQA')\n        for i, f in enumerate(RESOURCES):\n            json_file = f.file_name[:-3]\n            file_path = os.path.join(fpath, json_file)\n\n            with open(file_path, 'r') as infile:\n                data = infile.read()\n                new_data = data.replace('}\\n{', '},\\n{')\n                json_data = json.loads(f'[{new_data}]')\n\n            for ep in json_data:\n                # First 20 datasets have a different format than those later\n                if i < 21:\n                    self.data.append([create_entry_single(ep)])\n                else:\n                    self.data += create_entry_multiple(ep)\n\n    def get(self, episode_idx, entry_idx=0):\n        ep = self.data[episode_idx]\n        entry = ep[entry_idx]\n        action = dict()\n        action['id'] = episode_idx\n        for i, key in enumerate(self.existing_keys):\n            if i < 2:\n                continue\n            action[key] = entry[i]\n        action['episode_done'] = True\n        action['text'] = entry[0]\n        action['labels'] = [entry[1]]\n\n        return action\n\n    def share(self):\n        shared = super().share()\n        shared['data'] = self.data\n        return shared\n", "path": "parlai/tasks/amazon_qa/agents.py"}], "after_files": [{"content": "#!/usr/bin/env python3\n\n# Copyright (c) Facebook, Inc. and its affiliates.\n# This source code is licensed under the MIT license found in the\n# LICENSE file in the root directory of this source tree.\n\nfrom parlai.core.teachers import FixedDialogTeacher\nfrom .build import build, RESOURCES\nimport os\nimport json\n\n\nEXISTING_KEYS = [\n    'question',\n    'answer',\n    'asin',\n    'questionType',\n    'questionTime',\n    'askerID',\n    'answerType',\n    'answerTime',\n    'unixTime',\n    'answererID',\n    'helpful',\n    'answerScore',\n]\n\n\nclass DefaultTeacher(FixedDialogTeacher):\n    def __init__(self, opt, shared=None):\n        # store datatype\n        super().__init__(opt, shared)\n\n        dt = opt['datatype'].split(':')[0]\n        if dt != 'train':\n            raise RuntimeError('Not valid datatype (only train).')\n\n        if shared:\n            self.data = shared['data']\n            self.num_ex = shared['num_ex']\n            self.num_ep = shared['num_ep']\n        else:\n            build(opt)\n            self._setup_data()\n            self.num_ex = sum([len(x) for x in self.data])\n            self.num_ep = len(self.data)\n        self.reset()\n\n    def num_episodes(self):\n        return self.num_ep\n\n    def num_examples(self):\n        return self.num_ex\n\n    def _setup_data(self):\n        self.data = []\n\n        def create_entry_single(episode):\n            entry = []\n            for key in EXISTING_KEYS:\n                if key in episode:\n                    entry.append(str(episode[key]))\n                else:\n                    entry.append('N/A')\n            return entry\n\n        def create_entry_multiple(episode):\n            entries = []\n\n            for question in episode['questions']:\n                new_episode = dict()\n                new_episode['asin'] = episode['asin']\n                new_episode['askerID'] = question['askerID']\n                new_episode['questionTime'] = question['questionTime']\n                new_episode['quesitonType'] = question['questionType']\n                new_episode['question'] = question['questionText']\n\n                for answer in question['answers']:\n                    answer.update(new_episode)\n                    answer['answer'] = answer['answerText']\n                    entries.append([create_entry_single(answer)])\n\n            return entries\n\n        fpath = os.path.join(self.opt['datapath'], 'AmazonQA')\n        for i, f in enumerate(RESOURCES):\n            json_file = f.file_name[:-3]\n            file_path = os.path.join(fpath, json_file)\n\n            with open(file_path, 'r') as infile:\n                data = infile.read()\n                new_data = data.replace('}\\n{', '},\\n{')\n                json_data = json.loads(f'[{new_data}]')\n\n            for ep in json_data:\n                # First 20 datasets have a different format than those later\n                if i < 21:\n                    self.data.append([create_entry_single(ep)])\n                else:\n                    self.data += create_entry_multiple(ep)\n\n    def get(self, episode_idx, entry_idx=0):\n        ep = self.data[episode_idx]\n        entry = ep[entry_idx]\n        action = dict()\n        action['id'] = episode_idx\n        for i, key in enumerate(EXISTING_KEYS):\n            if i < 2:\n                continue\n            action[key] = entry[i]\n        action['episode_done'] = True\n        action['text'] = entry[0]\n        action['labels'] = [entry[1]]\n\n        return action\n\n    def share(self):\n        shared = super().share()\n        shared['data'] = self.data\n        shared['num_ex'] = self.num_ex\n        shared['num_ep'] = self.num_ep\n        return shared\n", "path": "parlai/tasks/amazon_qa/agents.py"}]}