class_docstr: Wrapper class for loading HuggingFace IterableDataset for a BPE-based ASR model
class_docstr_tok: ["Wrapper","class","for","loading","HuggingFace","IterableDataset","for","a","BPE-based","ASR","model"]
code: def __init__(
      self,
      audio_key: str,
      text_key: str,
      sample_rate_key: str,
      hf_data_cfg: DictConfig,
      tokenizer: "nemo.collections.common.tokenizers.TokenizerSpec",
      sample_rate: int,
      augmentor: "nemo.collections.asr.parts.perturb.AudioAugmentor" = None,
      trim: bool = False,
      return_sample_id: bool = False,
      id_key: str | None = None,
      channel_selector: ChannelSelectorType | None = None,
      normalize_db: float | None = None,
      ref_channel: int | None = None,
      global_rank: int = 0,
      world_size: int = 0,
      shuffle_n: int = 0,
      shuffle_seed: Optional[int] = None,
      use_start_end_token: bool = True,
      normalize_text: bool = False,
      symbols_to_keep: Optional[str] = None,
  ) -> None:
      if (
          use_start_end_token
          and hasattr(tokenizer, "bos_id")
          and tokenizer.bos_id > 0
      ):
          bos_id = tokenizer.bos_id
      else:
          bos_id = None
  
      if (
          use_start_end_token
          and hasattr(tokenizer, "eos_id")
          and tokenizer.eos_id > 0
      ):
          eos_id = tokenizer.eos_id
      else:
          eos_id = None
  
      if hasattr(tokenizer, "pad_id") and tokenizer.pad_id > 0:
          pad_id = tokenizer.pad_id
      else:
          pad_id = 0
  
      class TokenizerWrapper:
          def __init__(self, tokenizer):
              if isinstance(
                  tokenizer,
                  tokenizers.aggregate_tokenizer.AggregateTokenizer,
              ):
                  self.is_aggregate = True
              else:
                  self.is_aggregate = False
              self._tokenizer = tokenizer
  
          def __call__(self, *args):
              if isinstance(args[0], List) and self.is_aggregate:
                  t = []
                  for span in args[0]:
                      t.extend(
                          self._tokenizer.text_to_ids(
                              span["str"], span["lang"]
                          )
                      )
                  return t
  
              t = self._tokenizer.text_to_ids(*args)
              return t
  
      super().__init__(
          audio_key=audio_key,
          text_key=text_key,
          sample_rate_key=sample_rate_key,
          hf_data_cfg=hf_data_cfg,
          parser=TokenizerWrapper(tokenizer),
          sample_rate=sample_rate,
          augmentor=augmentor,
          trim=trim,
          bos_id=bos_id,
          eos_id=eos_id,
          pad_id=pad_id,
          return_sample_id=return_sample_id,
          id_key=id_key,
          channel_selector=channel_selector,
          normalize_db=normalize_db,
          ref_channel=ref_channel,
          global_rank=global_rank,
          world_size=world_size,
          shuffle_n=shuffle_n,
          shuffle_seed=shuffle_seed,
          normalize_text=normalize_text,
          symbols_to_keep=symbols_to_keep,
      )
  
imports: from typing import Callable, Dict, List, Optional, Tuple, Union
  import datasets
  import torch
  from datasets import concatenate_datasets
  from datasets.distributed import split_dataset_by_node
  from omegaconf import DictConfig, ListConfig, open_dict
  from nemo.collections.asr.data.audio_to_text import _speech_collate_fn
  from nemo.collections.asr.parts.preprocessing.perturb import AudioAugmentor
  from nemo.collections.asr.parts.preprocessing.segment import AudioSegment
  from nemo.collections.asr.parts.utils.audio_utils import ChannelSelectorType
  from nemo.collections.common import tokenizers
  from nemo.collections.common.parts.preprocessing import parsers
  from nemo.core.classes import Dataset, IterableDataset
  from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 2 to create a new HFIterableAudioToBPEDataset object from inherited base classes: _HFIterableAudioTextDataset with example: obj = HFIterableAudioToBPEDataset(audio_key, text_key, sample_rate_key, hf_data_cfg, tokenizer, sample_rate, augmentor, trim, return_sample_id, id_key, channel_selector, normalize_db, ref_channel, global_rank, world_size, shuffle_n, shuffle_seed, use_start_end_token, normalize_text, symbols_to_keep)