class_docstr: Builder class for BertExample objects.
class_docstr_tok: ["Builder","class","for","BertExample","objects","."]
docstr: Converts given ASR-hypothesis(hyp) and candidate string(ref) to features(token ids, mask, segment ids, etc).
  
  Args:
      hyp: Hypothesis text.
      ref: Candidate customization variants divided by ';'
      tags: List of labels corresponding to each token of ASR-hypothesis or None when building an example during inference.
  Returns:
      Features (input_ids, input_mask, segment_ids, labels_mask, labels, hyp_tokens, token_start_indices)
  
  Note that this method is called both for character-based example and for word-based example (to split to subwords).
  
  Character-based example:
      hyp:  "a s t r o n o m e r s _ d i d i e _ s o m o n _ a n d _ t r i s t i a n _ g l l o"
      ref:  "d i d i e r _ s a u m o n;a s t r o n o m i e;t r i s t a n _ g u i l l o t;t r i s t e s s e;m o n a d e;c h r i s t i a n;a s t r o n o m e r;s o l o m o n;d i d i d i d i d i;m e r c y"
      tags: "0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 3 3 3 3 3 3 3 3 3 3 3 3 3"
  
      resulting token sequence:
          '[CLS]', 'a', 's', 't', 'r', 'o', 'n', 'o', 'm', 'e', 'r', 's', '_', 'd', 'i', ..., 'l', 'o', '[SEP]', 'd', 'i', 'd', 'i', 'e', 'r', '_', 's', 'a', 'u', 'm', 'o', 'n', ..., '[SEP]'
  
  Word-based example:
      hyp:  "astronomers didie somon and tristian gllo"
      ref:  "didier saumon;astronomie;tristan guillot;tristesse;monade;christian;astronomer;solomon;dididididi;mercy"
      tags: None (not used for word-based case)
  
      resulting token sequence: 
          '[CLS]', 'astronomers', 'did', '##ie', 'so', '##mon', 'and', 'tri', '##sti', '##an', 'g', '##llo', '[SEP]', 'did', '##ier', 'sa', '##um', '##on', '[SEP]', 'astro', '##no', '##mie', '[SEP]', 'tristan', 'gui', '##llo', '##t', '[SEP]', ..., '[SEP]', 'mercy', '[SEP]']
docstr_tok: ["Converts","given","ASR-hypothesis","(","hyp",")","and","candidate","string","(","ref",")","to","features","(","token","ids",",","mask",",","segment","ids",",","etc",")",".","Args",":","hyp",":","Hypothesis","text",".","ref",":","Candidate","customization","variants","divided","by","'",";","'","tags",":","List","of","labels","corresponding","to","each","token","of","ASR-hypothesis","or","None","when","building","an","example","during","inference",".","Returns",":","Features","(","input_ids",",","input_mask",",","segment_ids",",","labels_mask",",","labels",",","hyp_tokens",",","token_start_indices",")","Note","that","this","method","is","called","both","for","character-based","example","and","for","word-based","example","(","to","split","to","subwords",")",".","Character-based","example",":","hyp",":","``","a","s","t","r","o","n","o","m","e","r","s","_","d","i","d","i","e","_","s","o","m","o","n","_","a","n","d","_","t","r","i","s","t","i","a","n","_","g","l","l","o","''","ref",":","``","d","i","d","i","e","r","_","s","a","u","m","o","n",";","a","s","t","r","o","n","o","m","i","e",";","t","r","i","s","t","a","n","_","g","u","i","l","l","o","t",";","t","r","i","s","t","e","s","s","e",";","m","o","n","a","d","e",";","c","h","r","i","s","t","i","a","n",";","a","s","t","r","o","n","o","m","e","r",";","s","o","l","o","m","o","n",";","d","i","d","i","d","i","d","i","d","i",";","m","e","r","c","y","''","tags",":","``","0","0","0","0","0","0","0","0","0","0","0","0","1","1","1","1","1","1","1","1","1","1","1","0","0","0","0","0","3","3","3","3","3","3","3","3","3","3","3","3","3","''","resulting","token","sequence",":","'","[","CLS","]","'",",","'","a","'",",","'s","'",",","'t","'",",","'","r","'",",","'","o","'",",","'n","'",",","'","o","'",",","'m","'",",","'","e","'",",","'","r","'",",","'s","'",",","'","_","'",",","'d","'",",","'","i","'",",","...",",","'","l","'",",","'","o","'",",","'","[","SEP","]","'",",","'d","'",",","'","i","'",",","'d","'",",","'","i","'",",","'","e","'",",","'","r","'",",","'","_","'",",","'s","'",",","'","a","'",",","'","u","'",",","'m","'",",","'","o","'",",","'n","'",",","...",",","'","[","SEP","]","'","Word-based","example",":","hyp",":","``","astronomers","didie","somon","and","tristian","gllo","''","ref",":","``","didier","saumon",";","astronomie",";","tristan","guillot",";","tristesse",";","monade",";","christian",";","astronomer",";","solomon",";","dididididi",";","mercy","''","tags",":","None","(","not","used","for","word-based","case",")","resulting","token","sequence",":","'","[","CLS","]","'",",","'astronomers","'",",","'did","'",",","'","#","#","ie","'",",","'so","'",",","'","#","#","mon","'",",","'and","'",",","'tri","'",",","'","#","#","sti","'",",","'","#","#","an","'",",","'","g","'",",","'","#","#","llo","'",",","'","[","SEP","]","'",",","'did","'",",","'","#","#","ier","'",",","'sa","'",",","'","#","#","um","'",",","'","#","#","on","'",",","'","[","SEP","]","'",",","'astro","'",",","'","#","#","no","'",",","'","#","#","mie","'",",","'","[","SEP","]","'",",","'tristan","'",",","'gui","'",",","'","#","#","llo","'",",","'","#","#","t","'",",","'","[","SEP","]","'",",","...",",","'","[","SEP","]","'",",","'mercy","'",",","'","[","SEP","]","'","]"]
code: def _get_input_features(
      self, hyp: str, ref: str, tags: List[int]
  ) -> Tuple[
      List[int],
      List[int],
      List[int],
      List[int],
      List[int],
      List[str],
      List[int],
  ]:
      """Converts given ASR-hypothesis(hyp) and candidate string(ref) to features(token ids, mask, segment ids, etc).
  
      Args:
          hyp: Hypothesis text.
          ref: Candidate customization variants divided by ';'
          tags: List of labels corresponding to each token of ASR-hypothesis or None when building an example during inference.
      Returns:
          Features (input_ids, input_mask, segment_ids, labels_mask, labels, hyp_tokens, token_start_indices)
  
      Note that this method is called both for character-based example and for word-based example (to split to subwords).
  
      Character-based example:
          hyp:  "a s t r o n o m e r s _ d i d i e _ s o m o n _ a n d _ t r i s t i a n _ g l l o"
          ref:  "d i d i e r _ s a u m o n;a s t r o n o m i e;t r i s t a n _ g u i l l o t;t r i s t e s s e;m o n a d e;c h r i s t i a n;a s t r o n o m e r;s o l o m o n;d i d i d i d i d i;m e r c y"
          tags: "0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 3 3 3 3 3 3 3 3 3 3 3 3 3"
  
          resulting token sequence:
              '[CLS]', 'a', 's', 't', 'r', 'o', 'n', 'o', 'm', 'e', 'r', 's', '_', 'd', 'i', ..., 'l', 'o', '[SEP]', 'd', 'i', 'd', 'i', 'e', 'r', '_', 's', 'a', 'u', 'm', 'o', 'n', ..., '[SEP]'
  
      Word-based example:
          hyp:  "astronomers didie somon and tristian gllo"
          ref:  "didier saumon;astronomie;tristan guillot;tristesse;monade;christian;astronomer;solomon;dididididi;mercy"
          tags: None (not used for word-based case)
  
          resulting token sequence:
              '[CLS]', 'astronomers', 'did', '##ie', 'so', '##mon', 'and', 'tri', '##sti', '##an', 'g', '##llo', '[SEP]', 'did', '##ier', 'sa', '##um', '##on', '[SEP]', 'astro', '##no', '##mie', '[SEP]', 'tristan', 'gui', '##llo', '##t', '[SEP]', ..., '[SEP]', 'mercy', '[SEP]']
      """
  
      labels_mask = []
      labels = []
      if tags is None:
          hyp_tokens, token_start_indices = self._split_to_wordpieces(
              hyp.split()
          )
      else:
          (
              hyp_tokens,
              labels,
              token_start_indices,
          ) = self._split_to_wordpieces_with_labels(hyp.split(), tags)
      references = ref.split(";")
      all_ref_tokens = []
      all_ref_segment_ids = []
      for i in range(len(references)):
          ref_tokens, _ = self._split_to_wordpieces(
              references[i].split()
          )
          all_ref_tokens.extend(ref_tokens + ["[SEP]"])
          all_ref_segment_ids.extend([i + 1] * (len(ref_tokens) + 1))
  
      input_tokens = (
          ["[CLS]"] + hyp_tokens + ["[SEP]"] + all_ref_tokens
      )  # ends with [SEP]
      input_ids = self._tokenizer.convert_tokens_to_ids(input_tokens)
      input_mask = [1] * len(input_ids)
      segment_ids = (
          [0] + [0] * len(hyp_tokens) + [0] + all_ref_segment_ids
      )
      if len(input_ids) != len(segment_ids):
          raise ValueError(
              "len(input_ids)="
              + str(len(input_ids))
              + " is different from len(segment_ids)="
              + str(len(segment_ids))
          )
  
      if tags:
          labels_mask = (
              [0] + [1] * len(labels) + [0] + [0] * len(all_ref_tokens)
          )
          labels = [0] + labels + [0] + [0] * len(all_ref_tokens)
      return (
          input_ids,
          input_mask,
          segment_ids,
          labels_mask,
          labels,
          hyp_tokens,
          token_start_indices,
      )
  
file_path: NeMo/nemo/collections/nlp/data/spellchecking_asr_customization/bert_example.py
imports: import logging
  from collections import OrderedDict
  from os import path
  from typing import Dict, List, Optional, Tuple, Union
  from transformers import PreTrainedTokenizerBase
label_desc: Use image node_id 6 for calling the BertExampleBuilder obj's underlying member method code with example usage: obj._get_input_features(hyp, ref, tags) and returns: input_ids,  input_mask,  segment_ids,  labels_mask,  labels,  hyp_tokens,  token_start_indices