class_docstr: NVIDIA DALI pipeline that loads tensors via one or more manifest files where each line containing a sample descriptor in JSON,
  including audio files, transcripts, and durations (in seconds).
  Here's an example:
  {"audio_filepath": "/path/to/audio.wav", "text_filepath": "/path/to/audio.txt", "duration": 23.147}
  ...
  {"audio_filepath": "/path/to/audio.wav", "text": "the transcription", "offset": 301.75, "duration": 0.82, "utt":
  "utterance_id", "ctm_utt": "en_4156", "side": "A"}
  
  Args:
      manifest_filepath: Path to manifest file with the format described above. Can be comma-separated paths.
      device (str): Determines the device type to be used for preprocessing. Allowed values are: 'cpu', 'gpu'.
      batch_size (int): Number of samples in a batch.
      parser (str, callable): A str for an inbuilt parser, or a callable with signature f(str) -> List[int].
      sample_rate (int): Sample rate to resample loaded audio to.
      num_threads (int): Number of CPU processing threads to be created by the DALI pipeline.
      max_duration (float): Determines the maximum allowed duration, in seconds, of the loaded audio files.
      min_duration (float): Determines the minimum allowed duration, in seconds, of the loaded audio files.
      bos_id (int): Id of beginning of sequence symbol to append if not None
      eos_id (int): Id of end of sequence symbol to append if not None
      pad_id (int): Id used to pad the input. Defaults to 0 if not provided.
      trim (bool): If True, it will extract the nonsilent region of the loaded audio signal.
      shuffle (bool): If set to True, the dataset will shuffled after loading.
      drop_last (bool): If set to True, the last batch will be dropped if incomplete. This will be the case when the shard size is not divisible by the batch size.
                        If set to False and the size of dataset is not divisible by the batch size, then the last batch will be smaller.
      device_id (int): Index of the GPU to be used (local_rank). Only applicable when device == 'gpu'. Defaults to 0.
      global_rank (int): Worker rank, used for partitioning shards. Defaults to 0.
      world_size (int): Total number of processes, used for partitioning shards. Defaults to 1.
      preprocessor_cfg (DictConfig): Preprocessor configuration. Supports AudioToMelSpectrogramPreprocessor and AudioToMFCCPreprocessor.
      return_sample_id (bool): whether to return the sample_id as a part of each sample (not supported yet).
class_docstr_tok: ["NVIDIA","DALI","pipeline","that","loads","tensors","via","one","or","more","manifest","files","where","each","line","containing","a","sample","descriptor","in","JSON",",","including","audio","files",",","transcripts",",","and","durations","(","in","seconds",")",".","Here","'s","an","example",":","{","``","audio_filepath","''",":","``","\/path\/to\/audio.wav","''",",","``","text_filepath","''",":","``","\/path\/to\/audio.txt","''",",","``","duration","''",":","23.147","}","...","{","``","audio_filepath","''",":","``","\/path\/to\/audio.wav","''",",","``","text","''",":","``","the","transcription","''",",","``","offset","''",":","301.75",",","``","duration","''",":","0.82",",","``","utt","''",":","''","utterance_id","''",",","``","ctm_utt","''",":","``","en_4156","''",",","``","side","''",":","``","A","''","}","Args",":","manifest_filepath",":","Path","to","manifest","file","with","the","format","described","above",".","Can","be","comma-separated","paths",".","device","(","str",")",":","Determines","the","device","type","to","be","used","for","preprocessing",".","Allowed","values","are",":","'cpu","'",",","'gpu","'",".","batch_size","(","int",")",":","Number","of","samples","in","a","batch",".","parser","(","str",",","callable",")",":","A","str","for","an","inbuilt","parser",",","or","a","callable","with","signature","f","(","str",")","-",">","List","[","int","]",".","sample_rate","(","int",")",":","Sample","rate","to","resample","loaded","audio","to",".","num_threads","(","int",")",":","Number","of","CPU","processing","threads","to","be","created","by","the","DALI","pipeline",".","max_duration","(","float",")",":","Determines","the","maximum","allowed","duration",",","in","seconds",",","of","the","loaded","audio","files",".","min_duration","(","float",")",":","Determines","the","minimum","allowed","duration",",","in","seconds",",","of","the","loaded","audio","files",".","bos_id","(","int",")",":","Id","of","beginning","of","sequence","symbol","to","append","if","not","None","eos_id","(","int",")",":","Id","of","end","of","sequence","symbol","to","append","if","not","None","pad_id","(","int",")",":","Id","used","to","pad","the","input",".","Defaults","to","0","if","not","provided",".","trim","(","bool",")",":","If","True",",","it","will","extract","the","nonsilent","region","of","the","loaded","audio","signal",".","shuffle","(","bool",")",":","If","set","to","True",",","the","dataset","will","shuffled","after","loading",".","drop_last","(","bool",")",":","If","set","to","True",",","the","last","batch","will","be","dropped","if","incomplete",".","This","will","be","the","case","when","the","shard","size","is","not","divisible","by","the","batch","size",".","If","set","to","False","and","the","size","of","dataset","is","not","divisible","by","the","batch","size",",","then","the","last","batch","will","be","smaller",".","device_id","(","int",")",":","Index","of","the","GPU","to","be","used","(","local_rank",")",".","Only","applicable","when","device","==","'gpu","'",".","Defaults","to","0.","global_rank","(","int",")",":","Worker","rank",",","used","for","partitioning","shards",".","Defaults","to","0.","world_size","(","int",")",":","Total","number","of","processes",",","used","for","partitioning","shards",".","Defaults","to","1.","preprocessor_cfg","(","DictConfig",")",":","Preprocessor","configuration",".","Supports","AudioToMelSpectrogramPreprocessor","and","AudioToMFCCPreprocessor",".","return_sample_id","(","bool",")",":","whether","to","return","the","sample_id","as","a","part","of","each","sample","(","not","supported","yet",")","."]
code: def __next__(self):
      outputs = self._iter.next()
      assert len(outputs) == 1
      dali_out = outputs[0]
      manifest_indices = dali_out["manifest_indices"].numpy()
  
      out = {}
      out_names = [
          "processed_signal",
          "processed_signal_len",
          "audio",
          "audio_len",
      ]
      for out_name in out_names:
          if out_name in dali_out:
              out[out_name] = dali_out[out_name].detach().clone()
  
      text_tokens = []
      text_tokens_len = []
      max_len = 0
      batch_size = manifest_indices.shape[0]
      for i, manifest_index in enumerate(manifest_indices):
          if not self.is_tarred_dataset:
              # Loose-file dataset. Index is integer based.
              manifest_index = manifest_index[0]
              (
                  text,
                  text_length,
              ) = self.manifest_processor.process_text_by_id(
                  manifest_index
              )
          else:
              # Tarred-file dataset. Index is filename based.
              resolved_manifest_indices = (
                  manifest_index.tobytes().decode().split(":")
              )
              resolved_manifest_index = resolved_manifest_indices[
                  2
              ]  # we require just the filename segment
              resolved_manifest_index = os.path.splitext(
                  resolved_manifest_index
              )[
                  0
              ]  # we dont need file extension
              (
                  text,
                  text_length,
              ) = self.manifest_processor.process_text_by_file_id(
                  resolved_manifest_index
              )
  
          text_tokens_len.append(text_length)
          text_tokens.append(text)
          if text_length > max_len:
              max_len = text_length
  
      transcript_out = torch.full(
          [batch_size, max_len],
          fill_value=self.manifest_processor.pad_id,
          dtype=torch.long,
      )
      for i, n in enumerate(text_tokens_len):
          transcript_out[i, :n] = torch.tensor(
              text_tokens[i], dtype=torch.long
          )
      transcript_len_out = torch.tensor(
          text_tokens_len, dtype=torch.long
      )
  
      out["transcript"] = transcript_out
      out["transcript_len"] = transcript_len_out
      return DALIOutputs(out)
  
imports: import math
  import operator
  import os.path
  import time
  from collections.abc import Iterator
  from typing import Callable, List, Optional, Union
  import torch
  from omegaconf import DictConfig
  from nemo.collections.asr.data.audio_to_text import ASRManifestProcessor, expand_sharded_filepaths
  from nemo.collections.common.parts.preprocessing import parsers
  from nemo.utils import logging, model_utils
label_desc: Use image node_id 7 for calling the _AudioTextDALIDataset obj's underlying member method code with example usage: obj.__next__() and returns: DALIOutputs