docstr: Creates a dataset to train a CTC-based G2P models.
  
  Args:
      manifest_filepath: path to a .json manifest that contains "phoneme_field" and "grapheme_field"
      tokenizer_graphemes: tokenizer for graphemes
      tokenizer_phonemes: tokenizer for phonemes
      do_lower: set to True to lower case input graphemes
      labels: output labels (tokenizer_phonemes vocabulary)
      max_source_len: max length of the grapheme input sequence (examples exceeding len will be dropped)
      phoneme_field: name of the field in manifest_filepath for ground truth phonemes
      grapheme_field: name of the field in manifest_filepath for input grapheme text
      with_labels: set to True for training and False for inference
docstr_tok: ["Creates","a","dataset","to","train","a","CTC-based","G2P","models",".","Args",":","manifest_filepath",":","path","to","a",".json","manifest","that","contains","``","phoneme_field","''","and","``","grapheme_field","''","tokenizer_graphemes",":","tokenizer","for","graphemes","tokenizer_phonemes",":","tokenizer","for","phonemes","do_lower",":","set","to","True","to","lower","case","input","graphemes","labels",":","output","labels","(","tokenizer_phonemes","vocabulary",")","max_source_len",":","max","length","of","the","grapheme","input","sequence","(","examples","exceeding","len","will","be","dropped",")","phoneme_field",":","name","of","the","field","in","manifest_filepath","for","ground","truth","phonemes","grapheme_field",":","name","of","the","field","in","manifest_filepath","for","input","grapheme","text","with_labels",":","set","to","True","for","training","and","False","for","inference"]
code: def __init__(
      self,
      manifest_filepath: str,
      tokenizer_graphemes: PreTrainedTokenizerBase,
      tokenizer_phonemes: PreTrainedTokenizerBase,
      do_lower: bool = True,
      labels: List[str] = None,
      max_source_len: int = 512,
      phoneme_field: str = "text",
      grapheme_field: str = "text_graphemes",
      with_labels: bool = True,
  ):
      """
      Creates a dataset to train a CTC-based G2P models.
  
      Args:
          manifest_filepath: path to a .json manifest that contains "phoneme_field" and "grapheme_field"
          tokenizer_graphemes: tokenizer for graphemes
          tokenizer_phonemes: tokenizer for phonemes
          do_lower: set to True to lower case input graphemes
          labels: output labels (tokenizer_phonemes vocabulary)
          max_source_len: max length of the grapheme input sequence (examples exceeding len will be dropped)
          phoneme_field: name of the field in manifest_filepath for ground truth phonemes
          grapheme_field: name of the field in manifest_filepath for input grapheme text
          with_labels: set to True for training and False for inference
      """
      super().__init__()
  
      if not os.path.exists(manifest_filepath):
          raise ValueError(f"{manifest_filepath} not found")
  
      self.manifest = manifest_filepath
      self.tokenizer_graphemes = tokenizer_graphemes
      self.tokenizer_phonemes = tokenizer_phonemes
      self.max_source_len = max_source_len
      self.labels = labels
      self.labels_tkn2id = {l: i for i, l in enumerate(labels)}
      self.data = []
      self.pad_token = 0
      self.with_labels = with_labels
  
      removed_ctc_max = 0
      removed_source_max = 0
      with open(manifest_filepath, "r") as f_in:
          logging.debug(f"Loading dataset from: {manifest_filepath}")
          for i, line in enumerate(f_in):
              item = json.loads(line)
  
              if do_lower:
                  item[grapheme_field] = item[grapheme_field].lower()
  
              if isinstance(
                  self.tokenizer_graphemes, PreTrainedTokenizerBase
              ):
                  grapheme_tokens = self.tokenizer_graphemes(
                      item[grapheme_field]
                  )
                  grapheme_tokens_len = len(
                      grapheme_tokens["input_ids"]
                  )
              else:
                  grapheme_tokens = (
                      self.tokenizer_graphemes.text_to_ids(
                          item[grapheme_field]
                      )
                  )
                  grapheme_tokens_len = len(grapheme_tokens)
  
              if with_labels:
                  target_tokens = self.tokenizer_phonemes.text_to_ids(
                      item[phoneme_field]
                  )
                  target_len = len(target_tokens)
  
                  if target_len > grapheme_tokens_len:
                      removed_ctc_max += 1
                      continue
  
                  if grapheme_tokens_len > max_source_len:
                      removed_source_max += 1
                      continue
  
                  self.data.append(
                      {
                          "graphemes": item[grapheme_field],
                          "phonemes": item[phoneme_field],
                          "target": target_tokens,
                          "target_len": target_len,
                      }
                  )
              else:
                  if len(grapheme_tokens) > max_source_len:
                      item[grapheme_field] = item[grapheme_field][
                          :max_source_len
                      ]
                      removed_source_max += 1
                  self.data.append(
                      {
                          "graphemes": item[grapheme_field],
                      }
                  )
  
      logging.info(
          f"Removed {removed_ctc_max} examples on CTC constraint, {removed_source_max} examples based on max_source_len from {manifest_filepath}"
      )
  
file_path: NeMo/nemo/collections/tts/g2p/data/ctc.py
imports: import json
  import os
  from typing import List
  import torch
  from transformers import PreTrainedTokenizerBase
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new CTCG2PBPEDataset object from inherited base classes: Dataset with example: obj = CTCG2PBPEDataset(manifest_filepath, tokenizer_graphemes, tokenizer_phonemes, do_lower, labels, max_source_len, phoneme_field, grapheme_field, with_labels)