code: def test_special_tokenization(self):
      from parlai.core.dict import DictionaryAgent
      from parlai.core.params import ParlaiParser
      from parlai.torchscript.tokenizer import ScriptableDictionaryAgent
  
      SPECIAL = ["Q00", "Q01"]
      text = "Don't have a Q00, man! Have a Q01 instead."
  
      parser = ParlaiParser(False, False)
      DictionaryAgent.add_cmdline_args(parser)
      with testing_utils.tempdir() as tmp:
          opt = parser.parse_kwargs(
              dict_tokenizer="gpt2", dict_file=os.path.join(tmp, "dict")
          )
  
          orig_dict = DictionaryAgent(opt)
  
          orig_bpe = orig_dict.bpe
          fused_key_bpe_ranks = {
              "\n".join(key): float(val)
              for key, val in orig_bpe.bpe_ranks.items()
          }
  
          sda = ScriptableDictionaryAgent(
              null_token=orig_dict.null_token,
              end_token=orig_dict.end_token,
              unk_token=orig_dict.unk_token,
              start_token=orig_dict.start_token,
              freq=orig_dict.freq,
              tok2ind=orig_dict.tok2ind,
              ind2tok=orig_dict.ind2tok,
              bpe_add_prefix_space=False,
              bpe_encoder=orig_bpe.encoder,
              bpe_byte_encoder=orig_bpe.byte_encoder,
              fused_key_bpe_ranks=fused_key_bpe_ranks,
              special_tokens=[],
              subword_bpe_version=(0, 0),
              fused_bpe_codes={},
              subword_bpe_separator="",
          )
  
          tokenized = sda.txt2vec(text, dict_tokenizer="gpt2")
          assert len(tokenized) == 15
          assert sda.vec2txt(tokenized, dict_tokenizer="gpt2") == text
  
          orig_dict = DictionaryAgent(opt)
          orig_dict.add_additional_special_tokens(SPECIAL)
          orig_bpe = orig_dict.bpe
          sda = ScriptableDictionaryAgent(
              null_token=orig_dict.null_token,
              end_token=orig_dict.end_token,
              unk_token=orig_dict.unk_token,
              start_token=orig_dict.start_token,
              freq=orig_dict.freq,
              tok2ind=orig_dict.tok2ind,
              ind2tok=orig_dict.ind2tok,
              bpe_add_prefix_space=False,
              bpe_encoder=orig_bpe.encoder,
              bpe_byte_encoder=orig_bpe.byte_encoder,
              fused_key_bpe_ranks=fused_key_bpe_ranks,
              special_tokens=SPECIAL,
              subword_bpe_version=(0, 0),
              fused_bpe_codes={},
              subword_bpe_separator="",
          )
  
          special_tokenized = sda.txt2vec(text, dict_tokenizer="gpt2")
          assert len(special_tokenized) == 15
          assert (
              sda.vec2txt(special_tokenized, dict_tokenizer="gpt2")
              == text
          )
          assert special_tokenized != tokenized
  
file_path: ParlAI/tests/nightly/gpu/test_torchscript.py
imports: import os
  import regex
  import unittest
  import parlai.utils.testing
  from parlai.agents.repeat_label.repeat_label import RepeatLabelAgent
  from parlai.core.agents import create_agent
  from parlai.core.params import ParlaiParser
  from parlai.core.worlds import create_task
  from parlai.utils.bpe import Gpt2BpeHelper
label_desc: Use image node_id 3 for calling the TestTorchScript obj's underlying member method code with example usage: obj.test_special_tokenization() without return types