class_docstr: Wrapper of HuggingFace AutoTokenizer https://huggingface.co/transformers/model_doc/auto.html#autotokenizer.
class_docstr_tok: ["Wrapper","of","HuggingFace","AutoTokenizer","https",":","\/\/huggingface.co\/transformers\/model_doc\/auto.html","#","autotokenizer","."]
docstr:         Args:
              pretrained_model_name: corresponds to HuggingFace-AutoTokenizer's 'pretrained_model_name_or_path' input argument. 
                  For more details please refer to https://huggingface.co/transformers/_modules/transformers/tokenization_auto.html#AutoTokenizer.from_pretrained. 
                  The list of all supported models can be found here: ALL_PRETRAINED_CONFIG_ARCHIVE_MAP
              vocab_file: path to file with vocabulary which consists
                  of characters separated by '
  '.
              mask_token: mask token 
              bos_token: the beginning of sequence token
              eos_token: the end of sequence token. Usually equal to sep_token
              pad_token: token to use for padding
              sep_token: token used for separating sequences
              cls_token: class token. Usually equal to bos_token
              unk_token: token to use for unknown tokens
              use_fast: whether to use fast HuggingFace tokenizer
          
docstr_tok: ["Args",":","pretrained_model_name",":","corresponds","to","HuggingFace-AutoTokenizer","'s","'pretrained_model_name_or_path","'","input","argument",".","For","more","details","please","refer","to","https",":","\/\/huggingface.co\/transformers\/_modules\/transformers\/tokenization_auto.html","#","AutoTokenizer.from_pretrained",".","The","list","of","all","supported","models","can","be","found","here",":","ALL_PRETRAINED_CONFIG_ARCHIVE_MAP","vocab_file",":","path","to","file","with","vocabulary","which","consists","of","characters","separated","by","'","'",".","mask_token",":","mask","token","bos_token",":","the","beginning","of","sequence","token","eos_token",":","the","end","of","sequence","token",".","Usually","equal","to","sep_token","pad_token",":","token","to","use","for","padding","sep_token",":","token","used","for","separating","sequences","cls_token",":","class","token",".","Usually","equal","to","bos_token","unk_token",":","token","to","use","for","unknown","tokens","use_fast",":","whether","to","use","fast","HuggingFace","tokenizer"]
code: def __init__(
      self,
      pretrained_model_name: str,
      vocab_file: Optional[str] = None,
      merges_file: Optional[str] = None,
      mask_token: Optional[str] = None,
      bos_token: Optional[str] = None,
      eos_token: Optional[str] = None,
      pad_token: Optional[str] = None,
      sep_token: Optional[str] = None,
      cls_token: Optional[str] = None,
      unk_token: Optional[str] = None,
      use_fast: Optional[bool] = False,
  ):
      """
      Args:
          pretrained_model_name: corresponds to HuggingFace-AutoTokenizer's 'pretrained_model_name_or_path' input argument.
              For more details please refer to https://huggingface.co/transformers/_modules/transformers/tokenization_auto.html#AutoTokenizer.from_pretrained.
              The list of all supported models can be found here: ALL_PRETRAINED_CONFIG_ARCHIVE_MAP
          vocab_file: path to file with vocabulary which consists
              of characters separated by '\n'.
          mask_token: mask token
          bos_token: the beginning of sequence token
          eos_token: the end of sequence token. Usually equal to sep_token
          pad_token: token to use for padding
          sep_token: token used for separating sequences
          cls_token: class token. Usually equal to bos_token
          unk_token: token to use for unknown tokens
          use_fast: whether to use fast HuggingFace tokenizer
      """
      try:
          # this logic deals with different huggingface tokenizers having different positional args
          if vocab_file is None:
              self.tokenizer = AUTOTOKENIZER.from_pretrained(
                  pretrained_model_name_or_path=pretrained_model_name,
                  use_fast=use_fast,
              )
          elif merges_file is None:
              self.tokenizer = AUTOTOKENIZER.from_pretrained(
                  pretrained_model_name_or_path=pretrained_model_name,
                  vocab_file=vocab_file,
                  use_fast=use_fast,
              )
          else:
              self.tokenizer = AUTOTOKENIZER.from_pretrained(
                  pretrained_model_name_or_path=pretrained_model_name,
                  vocab_file=vocab_file,
                  merges_file=merges_file,
                  use_fast=use_fast,
              )
      except Exception as e:
          raise ValueError(
              f"Unable to instantiate HuggingFace AUTOTOKENIZER for {pretrained_model_name}. Exception: {e}"
          )
  
      self.original_vocab_size = len(self.tokenizer)
      special_tokens_dict = {}
  
      # # setting special tokens, by default the default model's special tokens will be preserved
      # # unless passes new values to the special tokens
      if unk_token is not None:
          special_tokens_dict["unk_token"] = unk_token
      if mask_token is not None:
          special_tokens_dict["mask_token"] = mask_token
      if pad_token is not None:
          special_tokens_dict["pad_token"] = pad_token
  
      # if the model does not have eos_token but has sep_token,
      # set eos_token = sep_token, and vice versa
      if sep_token is not None:
          special_tokens_dict["sep_token"] = sep_token
      elif (
          self.tokenizer.sep_token is None and self.tokenizer.eos_token
      ):
          special_tokens_dict["sep_token"] = self.tokenizer.eos_token
      if eos_token is not None:
          special_tokens_dict["eos_token"] = eos_token
      elif (
          self.tokenizer.eos_token is None and self.tokenizer.sep_token
      ):
          special_tokens_dict["eos_token"] = self.tokenizer.sep_token
  
      # if the model does not have bos_token but has cls_token,
      # set bos_token = cls_token, and vice versa
      if bos_token is not None:
          special_tokens_dict["bos_token"] = bos_token
      elif (
          self.tokenizer.bos_token is None and self.tokenizer.cls_token
      ):
          special_tokens_dict["bos_token"] = self.tokenizer.cls_token
      if cls_token is not None:
          special_tokens_dict["cls_token"] = cls_token
      elif (
          self.tokenizer.cls_token is None and self.tokenizer.bos_token
      ):
          special_tokens_dict["cls_token"] = self.tokenizer.bos_token
  
      new_tokens_in_vocab = []
      for token in [
          mask_token,
          bos_token,
          eos_token,
          pad_token,
          sep_token,
          cls_token,
          unk_token,
      ]:
          if (
              token is not None
              and token not in self.tokenizer.get_vocab()
          ):
              new_tokens_in_vocab.append(token)
  
      if len(new_tokens_in_vocab) > 0:
          """
          Special tokens that were not previously included in the tokenizer's vocabulary file will be added to
          the vocabulary and, as a result, the model should be resized, for example:
  
          # define your model
          pretrained_model_name = 'roberta-base'
          model = nemo_nlp.modules.get_lm_model(pretrained_model_name=pretrained_model_name)
  
          # define pretrained tokenizer
          tokenizer_default = nemo_nlp.modules.get_tokenizer(tokenizer_name=pretrained_model_name)
  
          special_tokens = {'bos_token': '<BOS>',
                            'cls_token': '<CSL>',
                            'additional_special_tokens': ['<MY_NER_TOKEN>', '<ANOTHER_TOKEN>']}
          tokenizer_default.add_special_tokens(special_tokens_dict=special_tokens)
  
          # resize your model so that the embeddings for newly added tokens are updated during training/finetuning
          model.resize_token_embeddings(tokenizer_default.vocab_size)
  
          See NLP_Tokenizers.ipynb for more details.
          """
          logging.warning(
              f"{new_tokens_in_vocab} \n will be added to the vocabulary.\n"
              f"Please resize your model accordingly, "
              f"see NLP_Tokenizers.ipynb for more details."
          )
      self.add_special_tokens(special_tokens_dict)
      self.space_sensitive = self.text_to_tokens(
          "x y"
      ) != self.text_to_tokens("x") + self.text_to_tokens("y")
  
imports: from typing import Optional
  from transformers import AutoTokenizer
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new AutoTokenizer object from inherited base classes: TokenizerSpec with example: obj = AutoTokenizer(pretrained_model_name, vocab_file, merges_file, mask_token, bos_token, eos_token, pad_token, sep_token, cls_token, unk_token, use_fast)