class_docstr: Creates a dataset to train a T5G2P model.
class_docstr_tok: ["Creates","a","dataset","to","train","a","T5G2P","model","."]
docstr: Dataset to train T5-based G2P generative model.
  
  Args:
      manifest_filepath: path to a .json manifest that contains "phoneme_field" and "grapheme_field"
      tokenizer: pretrained T5 tokenizer
      max_source_len: max length of the grapheme input sequence (examples exceeding len will be dropped)
      max_target_len: max length of the phoneme sequence (examples exceeding len will be dropped)
      do_lower: a flag that indicates whether to lower case input grapheme sequence
      phoneme_field: name of the field in manifest_filepath for ground truth phonemes
      grapheme_field: name of the field in manifest_filepath for input grapheme text
      with_labels: set to True for training and False for inference
docstr_tok: ["Dataset","to","train","T5-based","G2P","generative","model",".","Args",":","manifest_filepath",":","path","to","a",".json","manifest","that","contains","``","phoneme_field","''","and","``","grapheme_field","''","tokenizer",":","pretrained","T5","tokenizer","max_source_len",":","max","length","of","the","grapheme","input","sequence","(","examples","exceeding","len","will","be","dropped",")","max_target_len",":","max","length","of","the","phoneme","sequence","(","examples","exceeding","len","will","be","dropped",")","do_lower",":","a","flag","that","indicates","whether","to","lower","case","input","grapheme","sequence","phoneme_field",":","name","of","the","field","in","manifest_filepath","for","ground","truth","phonemes","grapheme_field",":","name","of","the","field","in","manifest_filepath","for","input","grapheme","text","with_labels",":","set","to","True","for","training","and","False","for","inference"]
code: def __init__(
      self,
      manifest_filepath: str,
      tokenizer: PreTrainedTokenizerBase,
      max_source_len: int = 512,
      max_target_len: int = 512,
      do_lower: bool = False,
      grapheme_field: str = "text_graphemes",
      phoneme_field: str = "text",
      with_labels: bool = True,
  ):
      """
      Dataset to train T5-based G2P generative model.
  
      Args:
          manifest_filepath: path to a .json manifest that contains "phoneme_field" and "grapheme_field"
          tokenizer: pretrained T5 tokenizer
          max_source_len: max length of the grapheme input sequence (examples exceeding len will be dropped)
          max_target_len: max length of the phoneme sequence (examples exceeding len will be dropped)
          do_lower: a flag that indicates whether to lower case input grapheme sequence
          phoneme_field: name of the field in manifest_filepath for ground truth phonemes
          grapheme_field: name of the field in manifest_filepath for input grapheme text
          with_labels: set to True for training and False for inference
      """
      super().__init__()
  
      if not os.path.exists(manifest_filepath):
          raise ValueError(f"{manifest_filepath} not found")
  
      self.tokenizer = tokenizer
      self.max_source_len = max_source_len
      self.max_target_len = max_target_len
      self.do_lower = do_lower
      self.with_labels = with_labels
      self.data = []
  
      num_filtered = 0
  
      # Load grapheme/phoneme sequence pairs into self.data
      with open(manifest_filepath, "r") as f_in:
          logging.info(f"Loading dataset from: {manifest_filepath}")
          for line in f_in:
              item = json.loads(line)
              graphemes = item[grapheme_field]
              if do_lower:
                  graphemes = graphemes.lower()
  
              if with_labels:
                  graphemes_len = len(
                      self.tokenizer.tokenize(graphemes)
                  )
                  if graphemes_len > max_source_len:
                      num_filtered += 1
                      logging.debug(
                          f"dropping {graphemes_len} longer max_source_len"
                      )
                      continue
  
                  target_len = len(
                      self.tokenizer.tokenize(item[phoneme_field])
                  )
                  if max_target_len > 0 and target_len > max_target_len:
                      num_filtered += 1
                      logging.debug(
                          f"dropping {target_len} longer max_target_len"
                      )
                      continue
                  self.data.append(
                      {
                          "graphemes": graphemes,
                          "phonemes": item[phoneme_field],
                      }
                  )
              else:
                  # truncate input graphemes for inference if the length exceeds max_source_len
                  graphemes_tokenized = self.tokenizer(graphemes)[
                      "input_ids"
                  ]
                  if len(graphemes_tokenized) > max_source_len:
                      # -1 for special token at the end, <\s>
                      graphemes_tokenized_truncated = (
                          graphemes_tokenized[: max_source_len - 1]
                      )
                      graphemes = self.tokenizer.batch_decode(
                          [graphemes_tokenized_truncated]
                      )[0]
                  self.data.append({"graphemes": graphemes})
  
      logging.info(
          f"Filtered {num_filtered} too long entries from {manifest_filepath}."
      )
  
file_path: NeMo/nemo/collections/tts/g2p/data/t5.py
imports: import json
  import os
  import torch
  from transformers import PreTrainedTokenizerBase
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new T5G2PDataset object from inherited base classes: Dataset with example: obj = T5G2PDataset(manifest_filepath, tokenizer, max_source_len, max_target_len, do_lower, grapheme_field, phoneme_field, with_labels)