class_docstr: ASR BPE Mixin class that sets up a Tokenizer via a config
  
  This mixin class adds the method `_setup_tokenizer(...)`, which can be used by ASR models
  which depend on subword tokenization.
  
  The setup_tokenizer method adds the following parameters to the class -
      -   tokenizer_cfg: The resolved config supplied to the tokenizer (with `dir` and `type` arguments).
      -   tokenizer_dir: The directory path to the tokenizer vocabulary + additional metadata.
      -   tokenizer_type: The type of the tokenizer. Currently supports `bpe` and `wpe`, as well as `agg`.
      -   vocab_path: Resolved path to the vocabulary text file.
  
  In addition to these variables, the method will also instantiate and preserve a tokenizer
  (subclass of TokenizerSpec) if successful, and assign it to self.tokenizer.
  
  The mixin also supports aggregate tokenizers, which consist of ordinary, monolingual tokenizers.
  If a conversion between a monolongual and an aggregate tokenizer (or vice versa) is detected,
  all registered artifacts will be cleaned up.
code: def _setup_aggregate_tokenizer(self, tokenizer_cfg: DictConfig):
      # Prevent tokenizer parallelism (unless user has explicitly set it)
      if "TOKENIZERS_PARALLELISM" not in os.environ:
          os.environ["TOKENIZERS_PARALLELISM"] = "false"
  
      self.tokenizer_cfg = OmegaConf.to_container(
          tokenizer_cfg, resolve=True
      )  # type: dict
  
      # the aggregate tokenizer does not have one tokenizer_dir but multiple ones
      self.tokenizer_dir = None
  
      self.tokenizer_cfg.pop(
          "dir", None
      )  # Remove tokenizer directory, if any
      # Remove tokenizer_type -- obviously if we are here, the type is 'agg'
      self.tokenizer_type = self.tokenizer_cfg.pop("type").lower()
  
      # the aggregate tokenizer should not have these
      self.hf_tokenizer_kwargs = {}
      self.tokenizer_cfg.pop(
          "hf_kwargs", {}
      )  # Remove HF tokenizer kwargs, if any
  
      logging.info("_setup_tokenizer: detected an aggregate tokenizer")
      # need to de-register any monolingual config items if they exist
      self._cleanup_monolingual_and_aggregate_config_and_artifacts_if_needed()
  
      # overwrite tokenizer type
      if hasattr(self, "cfg") and "tokenizer" in self.cfg:
          self.cfg.tokenizer.type = self.tokenizer_type
  
      tokenizers_dict = {}
      # init each of the monolingual tokenizers found in the config and assemble into  AggregateTokenizer
      for lang, tokenizer_config in self.tokenizer_cfg[
          self.AGGREGATE_TOKENIZERS_DICT_PREFIX
      ].items():
          (
              tokenizer,
              model_path,
              vocab_path,
              spe_vocab_path,
          ) = self._make_tokenizer(tokenizer_config, lang)
  
          tokenizers_dict[lang] = tokenizer
          if hasattr(self, "cfg"):
              with open_dict(self.cfg.tokenizer):
                  self.cfg.tokenizer[
                      self.AGGREGATE_TOKENIZERS_DICT_PREFIX
                  ][lang]["dir"] = self.tokenizer_cfg[
                      self.AGGREGATE_TOKENIZERS_DICT_PREFIX
                  ][
                      lang
                  ][
                      "dir"
                  ]
                  self.cfg.tokenizer[
                      self.AGGREGATE_TOKENIZERS_DICT_PREFIX
                  ][lang]["type"] = self.tokenizer_cfg[
                      self.AGGREGATE_TOKENIZERS_DICT_PREFIX
                  ][
                      lang
                  ][
                      "type"
                  ]
  
      self.tokenizer = tokenizers.AggregateTokenizer(tokenizers_dict)
  
imports: import os
  from abc import ABC, abstractmethod
  from typing import List
  import torch
  from omegaconf import DictConfig, OmegaConf, open_dict
  import nemo.collections.asr.models
  from nemo.collections.asr.parts.mixins.asr_adapter_mixins import ASRAdapterModelMixin
  from nemo.collections.asr.parts.mixins.streaming import StreamingEncoder
  from nemo.collections.asr.parts.utils import asr_module_utils
  from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis
  from nemo.collections.common import tokenizers
  from nemo.utils import logging
label_desc: Use image node_id 3 for calling the ASRBPEMixin obj's underlying member method code with example usage: obj._setup_aggregate_tokenizer(tokenizer_cfg) without return types