class_docstr: This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
  created with the [`AutoTokenizer.from_pretrained`] class method.
  
  This class cannot be instantiated directly using `__init__()` (throws an error).
class_docstr_tok: ["This","is","a","generic","tokenizer","class","that","will","be","instantiated","as","one","of","the","tokenizer","classes","of","the","library","when","created","with","the","[","`","AutoTokenizer.from_pretrained","`","]","class","method",".","This","class","can","not","be","instantiated","directly","using","`","__init__","(",")","`","(","throws","an","error",")","."]
docstr: Register a new tokenizer in this mapping.
  
  
  Args:
      config_class ([`PretrainedConfig`]):
          The configuration corresponding to the model to register.
      slow_tokenizer_class ([`PretrainedTokenizer`], *optional*):
          The slow tokenizer to register.
      fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*):
          The fast tokenizer to register.
docstr_tok: ["Register","a","new","tokenizer","in","this","mapping",".","Args",":","config_class","(","[","`","PretrainedConfig","`","]",")",":","The","configuration","corresponding","to","the","model","to","register",".","slow_tokenizer_class","(","[","`","PretrainedTokenizer","`","]",",","*","optional","*",")",":","The","slow","tokenizer","to","register",".","fast_tokenizer_class","(","[","`","PretrainedTokenizerFast","`","]",",","*","optional","*",")",":","The","fast","tokenizer","to","register","."]
code: def register(
      config_class,
      slow_tokenizer_class=None,
      fast_tokenizer_class=None,
      exist_ok=False,
  ):
      """
      Register a new tokenizer in this mapping.
  
  
      Args:
          config_class ([`PretrainedConfig`]):
              The configuration corresponding to the model to register.
          slow_tokenizer_class ([`PretrainedTokenizer`], *optional*):
              The slow tokenizer to register.
          fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*):
              The fast tokenizer to register.
      """
      if slow_tokenizer_class is None and fast_tokenizer_class is None:
          raise ValueError(
              "You need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_class"
          )
      if slow_tokenizer_class is not None and issubclass(
          slow_tokenizer_class, PreTrainedTokenizerFast
      ):
          raise ValueError(
              "You passed a fast tokenizer in the `slow_tokenizer_class`."
          )
      if fast_tokenizer_class is not None and issubclass(
          fast_tokenizer_class, PreTrainedTokenizer
      ):
          raise ValueError(
              "You passed a slow tokenizer in the `fast_tokenizer_class`."
          )
  
      if (
          slow_tokenizer_class is not None
          and fast_tokenizer_class is not None
          and issubclass(fast_tokenizer_class, PreTrainedTokenizerFast)
          and fast_tokenizer_class.slow_tokenizer_class
          != slow_tokenizer_class
      ):
          raise ValueError(
              "The fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not "
              "consistent with the slow tokenizer class you passed (fast tokenizer has "
              f"{fast_tokenizer_class.slow_tokenizer_class} and you passed {slow_tokenizer_class}. Fix one of those "
              "so they match!"
          )
  
      # Avoid resetting a set slow/fast tokenizer if we are passing just the other ones.
      if config_class in TOKENIZER_MAPPING._extra_content:
          existing_slow, existing_fast = TOKENIZER_MAPPING[config_class]
          if slow_tokenizer_class is None:
              slow_tokenizer_class = existing_slow
          if fast_tokenizer_class is None:
              fast_tokenizer_class = existing_fast
  
      TOKENIZER_MAPPING.register(
          config_class,
          (slow_tokenizer_class, fast_tokenizer_class),
          exist_ok=exist_ok,
      )
  
file_path: transformers/src/transformers/models/auto/tokenization_auto.py
imports: import importlib
  import json
  import os
  import warnings
  from collections import OrderedDict
  from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
  from ...configuration_utils import PretrainedConfig
  from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
  from ...tokenization_utils import PreTrainedTokenizer
  from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
  from ...utils import cached_file, extract_commit_hash, is_g2p_en_available, is_sentencepiece_available, is_tokenizers_available, logging
  from ..encoder_decoder import EncoderDecoderConfig
  from .auto_factory import _LazyAutoMapping
  from .configuration_auto import CONFIG_MAPPING_NAMES, AutoConfig, config_class_to_model_type, model_type_to_module_name, replace_list_option_in_docstrings
label_desc: Use image node_id 3 for calling the AutoTokenizer obj's underlying member method code with example usage: obj.register(config_class, slow_tokenizer_class, fast_tokenizer_class, exist_ok) without return types