class_docstr: Text processor for huggingface datasets, mimicing the behavior of 
  `nemo.collections.asr.data.audio_to_text.ASRManifestProcessor`. 
  Basic text cleaning is also supported.
  Args:
      parser: Str for a language specific preprocessor or a callable.
      bos_id: BOS token id to add to the beginning of the transcript.
      eos_id: EOS token id to add to the end of the transcript.
      pad_id: PAD token id to pad transcripts to the same length.
      normalize_text: If true, normalizes text in HFTextProcessor
      symbols_to_keep: If not None, only keeps symbols in this list when normalizing text
class_docstr_tok: ["Text","processor","for","huggingface","datasets",",","mimicing","the","behavior","of","`","nemo.collections.asr.data.audio_to_text.ASRManifestProcessor","`",".","Basic","text","cleaning","is","also","supported",".","Args",":","parser",":","Str","for","a","language","specific","preprocessor","or","a","callable",".","bos_id",":","BOS","token","id","to","add","to","the","beginning","of","the","transcript",".","eos_id",":","EOS","token","id","to","add","to","the","end","of","the","transcript",".","pad_id",":","PAD","token","id","to","pad","transcripts","to","the","same","length",".","normalize_text",":","If","true",",","normalizes","text","in","HFTextProcessor","symbols_to_keep",":","If","not","None",",","only","keeps","symbols","in","this","list","when","normalizing","text"]
code: def process_text(
      self, text: str, lang: Optional[str] = None
  ) -> List[int]:
      if self.normalize_text:
          text = text.lower()
          # only keep alphanumeric characters, spaces and symbols defined in self.symbols_to_keep
          text = "".join(
              [
                  c
                  for c in text
                  if c.isalnum()
                  or c.isspace()
                  or c in self.symbols_to_keep
              ]
          )
  
      if (
          hasattr(self.parser, "is_aggregate")
          and self.parser.is_aggregate
          and isinstance(text, str)
      ):
          if lang is not None:
              text_tokens = self.parser(text, lang)
          # for future use if want to add language bypass to audio_to_text classes
          # elif hasattr(parser, "lang") and parser.lang is not None:
          #    text_tokens = parser(text, parser.lang)
          else:
              raise ValueError(
                  "lang required in manifest when using aggregate tokenizers"
              )
      else:
          text_tokens = self.parser(text)
      text_tokens_length = len(text_tokens)
      if self.bos_id is not None:
          text_tokens = [self.bos_id] + text_tokens
          text_tokens_length += 1
      if self.eos_id is not None:
          text_tokens = text_tokens + [self.eos_id]
          text_tokens_length += 1
      return text_tokens, text_tokens_length
  
imports: from typing import Callable, Dict, List, Optional, Tuple, Union
  import datasets
  import torch
  from datasets import concatenate_datasets
  from datasets.distributed import split_dataset_by_node
  from omegaconf import DictConfig, ListConfig, open_dict
  from nemo.collections.asr.data.audio_to_text import _speech_collate_fn
  from nemo.collections.asr.parts.preprocessing.perturb import AudioAugmentor
  from nemo.collections.asr.parts.preprocessing.segment import AudioSegment
  from nemo.collections.asr.parts.utils.audio_utils import ChannelSelectorType
  from nemo.collections.common import tokenizers
  from nemo.collections.common.parts.preprocessing import parsers
  from nemo.core.classes import Dataset, IterableDataset
  from nemo.core.neural_types import AudioSignal, LabelsType, LengthsType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 2 for calling the HFTextProcessor obj's underlying member method code with example usage: obj.process_text(text, lang) and returns: text_tokens,  text_tokens_length