docstr: Test TorchScriptable code for splitting tokens against reference GPT-2 version.
docstr_tok: ["Test","TorchScriptable","code","for","splitting","tokens","against","reference","GPT-2","version","."]
code: def test_gpt2_token_splitter(self):
      """
      Test TorchScriptable code for splitting tokens against reference GPT-2 version.
      """
  
      from parlai.scripts.torchscript import TorchScript
      from parlai.torchscript.tokenizer import ScriptableGpt2BpeHelper
  
      # Params
      tasks = ["taskmaster2", "convai2"]
      compiled_pattern = regex.compile(Gpt2BpeHelper.PATTERN)
  
      for task in tasks:
          opt = TorchScript.setup_args().parse_kwargs(
              task=task, datatype="train:ordered"
          )
          agent = RepeatLabelAgent(opt)
          # TODO(roller): make a proper create_teacher helper
          teacher = create_task(opt, agent).get_task_agent()
          num_examples = teacher.num_examples()
  
          print(
              f"\nStarting to test {num_examples:d} examples for the "
              f"{task} task."
          )
          for idx, message in enumerate(teacher):
              if idx % 10000 == 0:
                  print(f"Testing example #{idx:d}.")
              text = message["text"]
              canonical_tokens = regex.findall(compiled_pattern, text)
              scriptable_tokens = ScriptableGpt2BpeHelper.findall(text)
              self.assertEqual(canonical_tokens, scriptable_tokens)
              if idx + 1 == num_examples:
                  break
  
file_path: ParlAI/tests/nightly/gpu/test_torchscript.py
imports: import os
  import regex
  import unittest
  import parlai.utils.testing
  from parlai.agents.repeat_label.repeat_label import RepeatLabelAgent
  from parlai.core.agents import create_agent
  from parlai.core.params import ParlaiParser
  from parlai.core.worlds import create_task
  from parlai.utils.bpe import Gpt2BpeHelper
label_desc: Use image node_id 1 for calling the TestTorchScript obj's underlying member method code with example usage: obj.test_gpt2_token_splitter() without return types