class_docstr: ASR BPE Mixin class that sets up a Tokenizer via a config
  
  This mixin class adds the method `_setup_tokenizer(...)`, which can be used by ASR models
  which depend on subword tokenization.
  
  The setup_tokenizer method adds the following parameters to the class -
      -   tokenizer_cfg: The resolved config supplied to the tokenizer (with `dir` and `type` arguments).
      -   tokenizer_dir: The directory path to the tokenizer vocabulary + additional metadata.
      -   tokenizer_type: The type of the tokenizer. Currently supports `bpe` and `wpe`, as well as `agg`.
      -   vocab_path: Resolved path to the vocabulary text file.
  
  In addition to these variables, the method will also instantiate and preserve a tokenizer
  (subclass of TokenizerSpec) if successful, and assign it to self.tokenizer.
  
  The mixin also supports aggregate tokenizers, which consist of ordinary, monolingual tokenizers.
  If a conversion between a monolongual and an aggregate tokenizer (or vice versa) is detected,
  all registered artifacts will be cleaned up.
code: def _make_tokenizer(self, tokenizer_cfg: DictConfig, lang=None):
      tokenizer_type = tokenizer_cfg.get("type").lower()
      tokenizer_dir = tokenizer_cfg.get("dir")
  
      if tokenizer_type not in ["bpe", "wpe"]:
          raise ValueError(
              "`tokenizer.type` must be either `bpe` for SentencePiece tokenizer or"
              "`wpe` for BERT based tokenizer"
          )
  
      # defaults
      model_path = None
      vocab_path = None
      spe_vocab_path = None
  
      if tokenizer_type == "bpe":
          # This is a BPE Tokenizer
          if "model_path" in tokenizer_cfg:
              model_path = tokenizer_cfg.get("model_path")
          else:
              model_path = os.path.join(
                  tokenizer_dir, "tokenizer.model"
              )
  
          model_path = self.register_artifact(
              "tokenizer."
              + self.AGGREGATE_TOKENIZERS_DICT_PREFIX
              + "."
              + lang
              + ".model_path",
              model_path,
          )
  
          if "special_tokens" in tokenizer_cfg:
              special_tokens = tokenizer_cfg["special_tokens"]
              if special_tokens is not None:
                  raise ValueError(
                      "`special_tokens` are no longer supported for SentencePiece based tokenizers."
                  )
  
          # Update special tokens
          tokenizer = tokenizers.SentencePieceTokenizer(
              model_path=model_path
          )
  
          if "vocab_path" in tokenizer_cfg:
              vocab_path = tokenizer_cfg.get("vocab_path")
          else:
              vocab_path = os.path.join(tokenizer_dir, "vocab.txt")
  
          vocab_path = self.register_artifact(
              "tokenizer."
              + self.AGGREGATE_TOKENIZERS_DICT_PREFIX
              + "."
              + lang
              + ".vocab_path",
              vocab_path,
          )
  
          try:
              if "spe_tokenizer_vocab" in tokenizer_cfg:
                  spe_vocab_path = tokenizer_cfg.get(
                      "spe_tokenizer_vocab"
                  )
              else:
                  spe_vocab_path = os.path.join(
                      tokenizer_dir, "tokenizer.vocab"
                  )
  
              spe_vocab_path = self.register_artifact(
                  "tokenizer."
                  + self.AGGREGATE_TOKENIZERS_DICT_PREFIX
                  + "."
                  + lang
                  + ".spe_tokenizer_vocab",
                  spe_vocab_path,
              )
  
          except FileNotFoundError:
              # fallback case for older checkpoints that did not preserve the tokenizer.vocab
              spe_vocab_path = None
  
          vocabulary = {}
          for i in range(tokenizer.vocab_size):
              piece = tokenizer.ids_to_tokens([i])
              piece = piece[0]
              vocabulary[piece] = i + 1
  
          # wrapper method to get vocabulary conveniently
          def get_vocab():
              return vocabulary
  
          # attach utility values to the tokenizer wrapper
          tokenizer.tokenizer.vocab_size = len(vocabulary)
          tokenizer.tokenizer.get_vocab = get_vocab
          tokenizer.tokenizer.all_special_tokens = (
              tokenizer.special_token_to_id
          )
  
      else:
          # This is a WPE Tokenizer
          # If path from previous registration exists, remove it
          if "vocab_path" in tokenizer_cfg:
              vocab_path = tokenizer_cfg.get("vocab_path")
          else:
              vocab_path = os.path.join(tokenizer_dir, "vocab.txt")
  
          vocab_path = self.register_artifact(
              "tokenizer."
              + self.AGGREGATE_TOKENIZERS_DICT_PREFIX
              + "."
              + lang
              + ".vocab_path",
              vocab_path,
          )
  
          # If path from previous registration exists, remove it
          if "vocab_path" in tokenizer_cfg:
              tokenizer_cfg.pop("vocab_path")
  
          hf_tokenizer_kwargs = tokenizer_cfg.get("hf_kwargs", {})
          tokenizer = tokenizers.AutoTokenizer(
              pretrained_model_name=hf_tokenizer_kwargs.get(
                  "pretrained_model_name", "bert-base-cased"
              ),
              vocab_file=vocab_path,
              mask_token=hf_tokenizer_kwargs.get("mask_token", None),
              bos_token=hf_tokenizer_kwargs.get("bos_token", None),
              eos_token=hf_tokenizer_kwargs.get("eos_token", None),
              pad_token=hf_tokenizer_kwargs.get("pad_token", None),
              sep_token=hf_tokenizer_kwargs.get("sep_token", None),
              cls_token=hf_tokenizer_kwargs.get("cls_token", None),
              unk_token=hf_tokenizer_kwargs.get("unk_token", None),
              use_fast=hf_tokenizer_kwargs.get("use_fast", False),
          )
  
      logging.info(
          "Tokenizer {} initialized with {} tokens".format(
              tokenizer.__class__.__name__, tokenizer.vocab_size
          )
      )
  
      return tokenizer, model_path, vocab_path, spe_vocab_path
  
imports: import os
  from abc import ABC, abstractmethod
  from typing import List
  import torch
  from omegaconf import DictConfig, OmegaConf, open_dict
  import nemo.collections.asr.models
  from nemo.collections.asr.parts.mixins.asr_adapter_mixins import ASRAdapterModelMixin
  from nemo.collections.asr.parts.mixins.streaming import StreamingEncoder
  from nemo.collections.asr.parts.utils import asr_module_utils
  from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis
  from nemo.collections.common import tokenizers
  from nemo.utils import logging
label_desc: Use image node_id 4 for calling the ASRBPEMixin obj's underlying member method code with example usage: obj._make_tokenizer(tokenizer_cfg, lang) and returns: tokenizer,  model_path,  vocab_path,  spe_vocab_path, vocabulary