code: def get_nmt_tokenizer(
      library: str = "yttm",
      model_name: Optional[str] = None,
      tokenizer_model: Optional[str] = None,
      vocab_file: Optional[str] = None,
      merges_file: Optional[str] = None,
      special_tokens: Optional[Dict[str, str]] = None,
      use_fast: Optional[bool] = False,
      bpe_dropout: Optional[float] = 0.0,
      r2l: Optional[bool] = False,
      legacy: Optional[bool] = False,
      delimiter: Optional[str] = None,
  ):
      """
      Args:
          model_name: if using a pretrained model from NeMo, HuggingFace, or Megatron
          tokenizer_model: tokenizer model file of sentencepiece or youtokentome
          special_tokens: dict of special tokens
          vocab_file: path to vocab file
          use_fast: (only for HuggingFace AutoTokenizer) set to True to use fast HuggingFace tokenizer
          bpe_dropout: (only supported by YTTM tokenizer) BPE dropout tries to corrupt the standard segmentation procedure
              of BPE to help model better learn word compositionality and become robust to segmentation errors.
              It has empirically been shown to improve inference time BLEU scores.
          r2l: Whether to return subword IDs from right to left
      """
      if special_tokens is None:
          special_tokens_dict = {}
      else:
          special_tokens_dict = special_tokens
  
      if (library != "byte-level") and (
          model_name is None
          and (
              tokenizer_model is None
              or not os.path.isfile(tokenizer_model)
          )
      ):
          raise ValueError(
              "No Tokenizer path provided or file does not exist!"
          )
  
      if library == "yttm":
          logging.info(
              f"Getting YouTokenToMeTokenizer with model: {tokenizer_model} with r2l: {r2l}."
          )
          return YouTokenToMeTokenizer(
              model_path=tokenizer_model,
              bpe_dropout=bpe_dropout,
              r2l=r2l,
          )
      elif library == "huggingface":
          logging.info(
              f"Getting HuggingFace AutoTokenizer with pretrained_model_name: {model_name}"
          )
          return AutoTokenizer(
              pretrained_model_name=model_name,
              vocab_file=vocab_file,
              merges_file=merges_file,
              **special_tokens_dict,
              use_fast=use_fast,
          )
      elif library == "sentencepiece":
          logging.info(
              f"Getting SentencePiece with model: {tokenizer_model}"
          )
          return nemo.collections.common.tokenizers.sentencepiece_tokenizer.SentencePieceTokenizer(
              model_path=tokenizer_model, legacy=legacy
          )
      elif library == "byte-level":
          logging.info(f"Using byte-level tokenization")
          return ByteLevelTokenizer(special_tokens_dict)
      elif library == "regex":
          logging.info(f"Using regex tokenization")
          return RegExTokenizer().load_tokenizer(
              regex_file=tokenizer_model, vocab_file=vocab_file
          )
      elif library == "megatron":
          if model_name in megatron_tokenizer_model_map:
              model_name = megatron_tokenizer_model_map[model_name]
          logging.info(
              f"Getting Megatron tokenizer for pretrained model name: {model_name}, custom vocab file: {vocab_file}, and merges file: {merges_file}"
          )
          return get_tokenizer(
              tokenizer_name=model_name,
              vocab_file=vocab_file,
              merges_file=merges_file,
          )
      elif library == "tabular":
          return TabularTokenizer(vocab_file, delimiter=delimiter)
      else:
          raise NotImplementedError(
              'Currently we only support "yttm", "huggingface", "sentencepiece", "megatron", and "byte-level" tokenizer'
              "libraries."
          )
  
file_path: NeMo/nemo/collections/nlp/modules/common/tokenizer_utils.py
imports: import os.path
  from dataclasses import MISSING, dataclass
  from typing import Dict, List, Optional
  import nemo
  from nemo.collections.common.tokenizers.bytelevel_tokenizers import ByteLevelTokenizer
  from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer
  from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer
  from nemo.collections.common.tokenizers.regex_tokenizer import RegExTokenizer
  from nemo.collections.common.tokenizers.tabular_tokenizer import TabularTokenizer
  from nemo.collections.common.tokenizers.word_tokenizer import WordTokenizer
  from nemo.collections.common.tokenizers.youtokentome_tokenizer import YouTokenToMeTokenizer
  from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import get_huggingface_pretrained_lm_models_list
  from nemo.collections.nlp.modules.common.lm_utils import get_pretrained_lm_models_list
  from nemo.collections.nlp.parts.nlp_overrides import HAVE_MEGATRON_CORE
  from nemo.utils import logging
label_desc: Use image node_id 3 for calling a global function with example usage: get_nmt_tokenizer(library, model_name, tokenizer_model, vocab_file, merges_file, special_tokens, use_fast, bpe_dropout, r2l, legacy, delimiter) and returns: YouTokenToMeTokenizer, AutoTokenizer, nemo, ByteLevelTokenizer, RegExTokenizer, get_tokenizer, TabularTokenizer