class_docstr: Wrapper class for loading HuggingFace IterableDataset and converts to NeMo compatible format. 
  Args:
      audio_key: key to access audio data from the dataset
      text_key: key to access text data from the dataset
      sample_rate_key: key to access sample rate data from the dataset
      hf_data_cfg: HuggingFace dataset config, all params in this config will be passed to `hf_datasets.load_dataset`
      parser: Str for a language specific preprocessor or a callable.
      augmentor: An instance of `nemo.collections.asr.parts.perturb.AudioAugmentor` to apply on audio.
      trim: If true, trims silence using `nemo.collections.asr.parts.preprocessing.segment.AudioSegment`
      bos_id: BOS token id to add to the beginning of the transcript.
      eos_id: EOS token id to add to the end of the transcript.
      pad_id: PAD token id to pad transcripts to the same length.
      return_sample_id: If true, returns sample id from the dataset.
      channel_selector: ChannelSelectorType, which channel(s) to use for audio.
      normalize_db: Target RMS value for audio normalization.
      ref_channel: Reference channel for normalization.
      id_key: key to access sample id from the dataset
      global_rank: global rank of the current worker
      world_size: total number of workers
      shuffle_n: buffer size for shuffling
      shuffle_seed: seed for shuffling
      normalize_text: If true, normalizes text in HFTextProcessor
      symbols_to_keep: If not None, only keeps symbols in this list when normalizing text
class_docstr_tok: ["Wrapper","class","for","loading","HuggingFace","IterableDataset","and","converts","to","NeMo","compatible","format",".","Args",":","audio_key",":","key","to","access","audio","data","from","the","dataset","text_key",":","key","to","access","text","data","from","the","dataset","sample_rate_key",":","key","to","access","sample","rate","data","from","the","dataset","hf_data_cfg",":","HuggingFace","dataset","config",",","all","params","in","this","config","will","be","passed","to","`","hf_datasets.load_dataset","`","parser",":","Str","for","a","language","specific","preprocessor","or","a","callable",".","augmentor",":","An","instance","of","`","nemo.collections.asr.parts.perturb.AudioAugmentor","`","to","apply","on","audio",".","trim",":","If","true",",","trims","silence","using","`","nemo.collections.asr.parts.preprocessing.segment.AudioSegment","`","bos_id",":","BOS","token","id","to","add","to","the","beginning","of","the","transcript",".","eos_id",":","EOS","token","id","to","add","to","the","end","of","the","transcript",".","pad_id",":","PAD","token","id","to","pad","transcripts","to","the","same","length",".","return_sample_id",":","If","true",",","returns","sample","id","from","the","dataset",".","channel_selector",":","ChannelSelectorType",",","which","channel","(","s",")","to","use","for","audio",".","normalize_db",":","Target","RMS","value","for","audio","normalization",".","ref_channel",":","Reference","channel","for","normalization",".","id_key",":","key","to","access","sample","id","from","the","dataset","global_rank",":","global","rank","of","the","current","worker","world_size",":","total","number","of","workers","shuffle_n",":","buffer","size","for","shuffling","shuffle_seed",":","seed","for","shuffling","normalize_text",":","If","true",",","normalizes","text","in","HFTextProcessor","symbols_to_keep",":","If","not","None",",","only","keeps","symbols","in","this","list","when","normalizing","text"]
code: def __init__(
      self,
      audio_key: str,
      text_key: str,
      sample_rate_key: str,
      hf_data_cfg: Union[DictConfig, ListConfig],
      parser: Union[str, Callable],
      sample_rate: int,
      augmentor: "nemo.collections.asr.parts.perturb.AudioAugmentor" = None,
      trim: bool = False,
      bos_id: Optional[int] = None,
      eos_id: Optional[int] = None,
      pad_id: int = 0,
      return_sample_id: bool = False,
      channel_selector: Optional[ChannelSelectorType] = None,
      normalize_db: Optional[float] = None,
      ref_channel: Optional[int] = None,
      id_key: Optional[str] = None,
      global_rank: int = 0,
      world_size: int = 0,
      shuffle_n: int = 0,
      shuffle_seed: Optional[int] = None,
      normalize_text: bool = False,
      symbols_to_keep: Optional[str] = None,
  ) -> None:
      super().__init__()
  
      if return_sample_id and id_key is None:
          raise ValueError(
              "return_sample_id is True, but id_key is None"
          )
  
      self.audio_key = audio_key
      self.text_key = text_key
      self.sample_rate_key = sample_rate_key
      self.id_key = id_key
      self.sample_rate = sample_rate
      self.augmentor = (
          augmentor if augmentor is not None else AudioAugmentor()
      )
      self.trim = trim
      self.return_sample_id = return_sample_id
      self.channel_selector = channel_selector
      self.normalize_db = normalize_db
      self.ref_channel = ref_channel
  
      self.text_processor = HFTextProcessor(
          parser,
          bos_id,
          eos_id,
          pad_id,
          normalize_text,
          symbols_to_keep,
      )
  
      data_config_list = (
          [hf_data_cfg]
          if isinstance(hf_data_cfg, DictConfig)
          else hf_data_cfg
      )
      dataset_list = []
      for data_cfg in data_config_list:
          with open_dict(data_cfg):
              if "streaming" in data_cfg and not data_cfg.streaming:
                  logging.warning(
                      "streaming must be True for streaming dataset, but you use streaming=False. Forcing streaming=True"
                  )
              # streaming must be True for iterable dataset
              data_cfg.streaming = True
          logging.info(
              f"Streaming HuggingFace IterableDataset with cfg: {data_cfg}"
          )
          dataset_list.append(hf_datasets.load_dataset(**data_cfg))
  
      self.dataset = concatenate_datasets(dataset_list)
      logging.info(
          f"Total number of samples cannot be extracted from HF streaming dataset"
      )
  
      if shuffle_n > 0:
          self.dataset = self.dataset.shuffle(
              seed=shuffle_seed, buffer_size=shuffle_n
          )
  
      self.dataset = split_dataset_by_node(
          self.dataset, global_rank, world_size
      )
      self.dataset = self.dataset.map(self._build_sample)
  
imports: from typing import Callable, Dict, List, Optional, Tuple, Union
  import datasets
  import torch
  from datasets import concatenate_datasets
  from datasets.distributed import split_dataset_by_node
  from omegaconf import DictConfig, ListConfig, open_dict
  from nemo.collections.asr.data.audio_to_text import _speech_collate_fn
  from nemo.collections.asr.parts.preprocessing.perturb import AudioAugmentor
  from nemo.collections.asr.parts.preprocessing.segment import AudioSegment
  from nemo.collections.asr.parts.utils.audio_utils import ChannelSelectorType
  from nemo.collections.common import tokenizers
  from nemo.collections.common.parts.preprocessing import parsers
  from nemo.core.classes import Dataset, IterableDataset
  from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new _HFIterableAudioTextDataset object from inherited base classes: IterableDataset with example: obj = _HFIterableAudioTextDataset(audio_key, text_key, sample_rate_key, hf_data_cfg, parser, sample_rate, augmentor, trim, bos_id, eos_id, pad_id, return_sample_id, channel_selector, normalize_db, ref_channel, id_key, global_rank, world_size, shuffle_n, shuffle_seed, normalize_text, symbols_to_keep)