class_docstr: Text processor for huggingface datasets, mimicing the behavior of 
  `nemo.collections.asr.data.audio_to_text.ASRManifestProcessor`. 
  Basic text cleaning is also supported.
  Args:
      parser: Str for a language specific preprocessor or a callable.
      bos_id: BOS token id to add to the beginning of the transcript.
      eos_id: EOS token id to add to the end of the transcript.
      pad_id: PAD token id to pad transcripts to the same length.
      normalize_text: If true, normalizes text in HFTextProcessor
      symbols_to_keep: If not None, only keeps symbols in this list when normalizing text
class_docstr_tok: ["Text","processor","for","huggingface","datasets",",","mimicing","the","behavior","of","`","nemo.collections.asr.data.audio_to_text.ASRManifestProcessor","`",".","Basic","text","cleaning","is","also","supported",".","Args",":","parser",":","Str","for","a","language","specific","preprocessor","or","a","callable",".","bos_id",":","BOS","token","id","to","add","to","the","beginning","of","the","transcript",".","eos_id",":","EOS","token","id","to","add","to","the","end","of","the","transcript",".","pad_id",":","PAD","token","id","to","pad","transcripts","to","the","same","length",".","normalize_text",":","If","true",",","normalizes","text","in","HFTextProcessor","symbols_to_keep",":","If","not","None",",","only","keeps","symbols","in","this","list","when","normalizing","text"]
code: def __init__(
      self,
      parser: Union[str, Callable],
      bos_id: Optional[int] = None,
      eos_id: Optional[int] = None,
      pad_id: int = 0,
      normalize_text: bool = False,
      symbols_to_keep: Optional[str | List[str]] = None,
  ):
      self.parser = parser
      self.eos_id = eos_id
      self.bos_id = bos_id
      self.pad_id = pad_id
      self.normalize_text = normalize_text
      self.symbols_to_keep = (
          [x for x in symbols_to_keep]
          if symbols_to_keep is not None
          else []
      )
  
imports: from typing import Callable, Dict, List, Optional, Tuple, Union
  import datasets
  import torch
  from datasets import concatenate_datasets
  from datasets.distributed import split_dataset_by_node
  from omegaconf import DictConfig, ListConfig, open_dict
  from nemo.collections.asr.data.audio_to_text import _speech_collate_fn
  from nemo.collections.asr.parts.preprocessing.perturb import AudioAugmentor
  from nemo.collections.asr.parts.preprocessing.segment import AudioSegment
  from nemo.collections.asr.parts.utils.audio_utils import ChannelSelectorType
  from nemo.collections.common import tokenizers
  from nemo.collections.common.parts.preprocessing import parsers
  from nemo.core.classes import Dataset, IterableDataset
  from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new HFTextProcessor object  with example: obj = HFTextProcessor(parser, bos_id, eos_id, pad_id, normalize_text, symbols_to_keep)