class_docstr: A `PretrainedTransformerTokenizer` uses a model from HuggingFace's
  `transformers` library to tokenize some input text.  This often means wordpieces
  (where `'AllenNLP is awesome'` might get split into `['Allen', '##NL', '##P', 'is',
  'awesome']`), but it could also use byte-pair encoding, or some other tokenization, depending
  on the pretrained model that you're using.
  
  We take a model name as an input parameter, which we will pass to
  `AutoTokenizer.from_pretrained`.
  
  We also add special tokens relative to the pretrained model and truncate the sequences.
  
  This tokenizer also indexes tokens and adds the indexes to the `Token` fields so that
  they can be picked up by `PretrainedTransformerIndexer`.
  
  Registered as a `Tokenizer` with name "pretrained_transformer".
  
  # Parameters
  
  model_name : `str`
      The name of the pretrained wordpiece tokenizer to use.
  add_special_tokens : `bool`, optional, (default=`True`)
      If set to `True`, the sequences will be encoded with the special tokens relative
      to their model.
  max_length : `int`, optional (default=`None`)
      If set to a number, will limit the total sequence returned so that it has a maximum length.
  tokenizer_kwargs: `Dict[str, Any]`, optional (default = `None`)
      Dictionary with
      [additional arguments](https://github.com/huggingface/transformers/blob/155c782a2ccd103cf63ad48a2becd7c76a7d2115/transformers/tokenization_utils.py#L691)
      for `AutoTokenizer.from_pretrained`.
  verification_tokens: `Tuple[str, str]`, optional (default = `None`)
      A pair of tokens having different token IDs. It's used for reverse-engineering special tokens.
code: def _reverse_engineer_special_tokens(
      self,
      token_a: str,
      token_b: str,
      model_name: str,
      tokenizer_kwargs: Optional[Dict[str, Any]],
  ):
      # storing the special tokens
      self.sequence_pair_start_tokens = []
      self.sequence_pair_mid_tokens = []
      self.sequence_pair_end_tokens = []
      # storing token type ids for the sequences
      self.sequence_pair_first_token_type_id = None
      self.sequence_pair_second_token_type_id = None
  
      # storing the special tokens
      self.single_sequence_start_tokens = []
      self.single_sequence_end_tokens = []
      # storing token type id for the sequence
      self.single_sequence_token_type_id = None
  
      # Reverse-engineer the tokenizer for two sequences
      from allennlp.common import cached_transformers
  
      tokenizer_with_special_tokens = cached_transformers.get_tokenizer(
          model_name,
          add_special_tokens=True,
          **(tokenizer_kwargs or {}),
      )
      dummy_output = tokenizer_with_special_tokens.encode_plus(
          token_a,
          token_b,
          add_special_tokens=True,
          return_token_type_ids=True,
          return_attention_mask=False,
      )
      if len(dummy_output["token_type_ids"]) != len(
          dummy_output["input_ids"]
      ):
          logger.warning(
              "Tokenizer library did not return valid token type ids. We will assume they are all zero."
          )
          dummy_output["token_type_ids"] = [0] * len(
              dummy_output["input_ids"]
          )
  
      dummy_a = self.tokenizer.encode(
          token_a, add_special_tokens=False
      )[0]
      assert dummy_a in dummy_output["input_ids"]
      dummy_b = self.tokenizer.encode(
          token_b, add_special_tokens=False
      )[0]
      assert dummy_b in dummy_output["input_ids"]
      assert dummy_a != dummy_b
  
      seen_dummy_a = False
      seen_dummy_b = False
      for token_id, token_type_id in zip(
          dummy_output["input_ids"], dummy_output["token_type_ids"]
      ):
          if token_id == dummy_a:
              if (
                  seen_dummy_a or seen_dummy_b
              ):  # seeing a twice or b before a
                  raise ValueError(
                      "Cannot auto-determine the number of special tokens added."
                  )
              seen_dummy_a = True
              assert (
                  self.sequence_pair_first_token_type_id is None
                  or self.sequence_pair_first_token_type_id
                  == token_type_id
              ), "multiple different token type ids found for the first sequence"
              self.sequence_pair_first_token_type_id = token_type_id
              continue
  
          if token_id == dummy_b:
              if seen_dummy_b:  # seeing b twice
                  raise ValueError(
                      "Cannot auto-determine the number of special tokens added."
                  )
              seen_dummy_b = True
              assert (
                  self.sequence_pair_second_token_type_id is None
                  or self.sequence_pair_second_token_type_id
                  == token_type_id
              ), "multiple different token type ids found for the second sequence"
              self.sequence_pair_second_token_type_id = token_type_id
              continue
  
          token = Token(
              tokenizer_with_special_tokens.convert_ids_to_tokens(
                  token_id
              ),
              text_id=token_id,
              type_id=token_type_id,
          )
          if not seen_dummy_a:
              self.sequence_pair_start_tokens.append(token)
          elif not seen_dummy_b:
              self.sequence_pair_mid_tokens.append(token)
          else:
              self.sequence_pair_end_tokens.append(token)
  
      assert (
          len(self.sequence_pair_start_tokens)
          + len(self.sequence_pair_mid_tokens)
          + len(self.sequence_pair_end_tokens)
      ) == self.tokenizer.num_special_tokens_to_add(pair=True)
  
      # Reverse-engineer the tokenizer for one sequence
      dummy_output = tokenizer_with_special_tokens.encode_plus(
          token_a,
          add_special_tokens=True,
          return_token_type_ids=True,
          return_attention_mask=False,
      )
      if len(dummy_output["token_type_ids"]) != len(
          dummy_output["input_ids"]
      ):
          logger.warning(
              "Tokenizer library did not return valid token type ids. We will assume they are all zero."
          )
          dummy_output["token_type_ids"] = [0] * len(
              dummy_output["input_ids"]
          )
  
      seen_dummy_a = False
      for token_id, token_type_id in zip(
          dummy_output["input_ids"], dummy_output["token_type_ids"]
      ):
          if token_id == dummy_a:
              if seen_dummy_a:
                  raise ValueError(
                      "Cannot auto-determine the number of special tokens added."
                  )
              seen_dummy_a = True
              assert (
                  self.single_sequence_token_type_id is None
                  or self.single_sequence_token_type_id == token_type_id
              ), "multiple different token type ids found for the sequence"
              self.single_sequence_token_type_id = token_type_id
              continue
  
          token = Token(
              tokenizer_with_special_tokens.convert_ids_to_tokens(
                  token_id
              ),
              text_id=token_id,
              type_id=token_type_id,
          )
          if not seen_dummy_a:
              self.single_sequence_start_tokens.append(token)
          else:
              self.single_sequence_end_tokens.append(token)
  
      assert (
          len(self.single_sequence_start_tokens)
          + len(self.single_sequence_end_tokens)
      ) == self.tokenizer.num_special_tokens_to_add(pair=False)
  
imports: import copy
  import dataclasses
  import logging
  from typing import Any, Dict, List, Optional, Tuple, Iterable
  from transformers import PreTrainedTokenizer
  from allennlp.common.util import sanitize_wordpiece
  from allennlp.data.tokenizers.token_class import Token
  from allennlp.data.tokenizers.tokenizer import Tokenizer
label_desc: Use image node_id 2 for calling the PretrainedTransformerTokenizer obj's underlying member method code with example usage: obj._reverse_engineer_special_tokens(token_a, token_b, model_name, tokenizer_kwargs) without return types