docstr: Creates dataset to use to run training and inference on G2PClassificationModel.
  Processes WikiHomograph raw data files:
  https://github.com/google-research-datasets/WikipediaHomographData/tree/master/data
  
  Args:
      manifest: path to manifest with "heteronym_span", "start_end", "text_graphemes"
          and (optional) "word_id" fields. "word_id" is required for model training.
      tokenizer: pretrained tokenizer
      heteronym_dict: a dictionary where each grapheme contains word_id to ipa_form mappings, e.g.,
          {'use': {'use_nou': "'juːs", 'use_vrb': "'juːz"}}
      wordid_to_idx: mapping from word id to index
      max_seq_len: maximum input sequence length
      grapheme_field: name of the field in the .json manifest with grapheme input
      with_labels: indicates whether labels are provided in the manifest. False for inference, True for training
docstr_tok: ["Creates","dataset","to","use","to","run","training","and","inference","on","G2PClassificationModel",".","Processes","WikiHomograph","raw","data","files",":","https",":","\/\/github.com\/google-research-datasets\/WikipediaHomographData\/tree\/master\/data","Args",":","manifest",":","path","to","manifest","with","``","heteronym_span","''",",","``","start_end","''",",","``","text_graphemes","''","and","(","optional",")","``","word_id","''","fields",".","``","word_id","''","is","required","for","model","training",".","tokenizer",":","pretrained","tokenizer","heteronym_dict",":","a","dictionary","where","each","grapheme","contains","word_id","to","ipa_form","mappings",",","e.g.",",","{","'use","'",":","{","'use_nou","'",":","``","'ju\u02d0s","''",",","'use_vrb","'",":","``","'ju\u02d0z","''","}","}","wordid_to_idx",":","mapping","from","word","id","to","index","max_seq_len",":","maximum","input","sequence","length","grapheme_field",":","name","of","the","field","in","the",".json","manifest","with","grapheme","input","with_labels",":","indicates","whether","labels","are","provided","in","the","manifest",".","False","for","inference",",","True","for","training"]
code: def __init__(
      self,
      manifest: str,
      tokenizer: TokenizerSpec,
      heteronym_dict: Dict[str, Dict[str, str]],
      wordid_to_idx: Dict[str, int],
      max_seq_len: int = 512,
      grapheme_field: str = "text_graphemes",
      with_labels: bool = True,
  ):
      """
      Creates dataset to use to run training and inference on G2PClassificationModel.
      Processes WikiHomograph raw data files:
      https://github.com/google-research-datasets/WikipediaHomographData/tree/master/data
  
      Args:
          manifest: path to manifest with "heteronym_span", "start_end", "text_graphemes"
              and (optional) "word_id" fields. "word_id" is required for model training.
          tokenizer: pretrained tokenizer
          heteronym_dict: a dictionary where each grapheme contains word_id to ipa_form mappings, e.g.,
              {'use': {'use_nou': "'juːs", 'use_vrb': "'juːz"}}
          wordid_to_idx: mapping from word id to index
          max_seq_len: maximum input sequence length
          grapheme_field: name of the field in the .json manifest with grapheme input
          with_labels: indicates whether labels are provided in the manifest. False for inference, True for training
      """
      super().__init__()
  
      if not os.path.exists(manifest):
          raise ValueError(f"{manifest} not found")
  
      self.tokenizer = tokenizer
      self.max_seq_len = max_seq_len
      self.data = []
      self.pad_token = 0
      self.with_labels = with_labels
      self.heteronym_dict = heteronym_dict
      self.wordid_to_idx = wordid_to_idx
      self.LOSS_PAD_TOKEN = -100
      self.PAD_TOKEN = 0
  
      num_skipped = 0
      with open(manifest, "r") as f:
          for line in f:
              line = json.loads(line)
              cur_start_end, cur_heteronyms = (
                  line["start_end"],
                  line["heteronym_span"],
              )
  
              # during inference word_id is not present in the manifest
              if "word_id" in line:
                  cur_word_ids = line["word_id"]
              else:
                  if isinstance(cur_heteronyms, str):
                      cur_word_ids = None
                  else:
                      cur_word_ids = [None] * len(cur_heteronyms)
  
              if isinstance(cur_heteronyms, str):
                  cur_start_end, cur_heteronyms, cur_word_ids = (
                      [cur_start_end],
                      [cur_heteronyms],
                      [cur_word_ids],
                  )
  
              example = self._prepare_sample(
                  line[grapheme_field],
                  cur_start_end,
                  cur_heteronyms,
                  cur_word_ids,
              )
              if example is None:
                  num_skipped += 1
              else:
                  example_dict = {
                      "input_ids": example[0],
                      "subtokens_mask": example[1],
                      "target": example[
                          2
                      ],  # None if self.with_labels is False
                  }
                  self.data.append(example_dict)
  
      logging.info(
          f"Number of samples in {manifest}: {len(self.data)}, remove {num_skipped} lines"
      )
  
file_path: NeMo/nemo/collections/tts/g2p/data/heteronym_classification.py
imports: import json
  import os
  import re
  from typing import Dict, List, Optional, Tuple
  import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new HeteronymClassificationDataset object from inherited base classes: Dataset with example: obj = HeteronymClassificationDataset(manifest, tokenizer, heteronym_dict, wordid_to_idx, max_seq_len, grapheme_field, with_labels)