class_docstr: Builder class for BertExample objects.
class_docstr_tok: ["Builder","class","for","BertExample","objects","."]
docstr: Constructs a BERT Example.
  
  Args:
      hyp: Hypothesis text.
      ref: Candidate customization variants divided by ';'
      target:
          if infer==False, string of labels (each label is 1-based index of correct candidate) or 0.
          if infer==True, it can be None or string of labels (each label is 1-based index of some candidate). In inference this can be used to get corresponding fragments to fragment_indices.
      span_info:
          string of format "CUSTOM 6 20;CUSTOM 40 51", number of parts corresponds to number of targets. Can be empty if target is 0.
          If infer==False, numbers are correct start and end(exclusive) positions of the corresponding target candidate in the text.
          If infer==True, numbers are EXPECTED positions in the text. In inference this can be used to get corresponding fragments to fragment_indices.
      infer: inference mode
  Returns:
      BertExample, or None if the conversion from text to tags was infeasible
  
  Example (infer=False):
      hyp: "a s t r o n o m e r s _ d i d i e _ s o m o n _ a n d _ t r i s t i a n _ g l l o"
      ref: "d i d i e r _ s a u m o n;a s t r o n o m i e;t r i s t a n _ g u i l l o t;t r i s t e s s e;m o n a d e;c h r i s t i a n;a s t r o n o m e r;s o l o m o n;d i d i d i d i d i;m e r c y"
      target: "1 3"
      span_info: "CUSTOM 12 23;CUSTOM 28 41"
docstr_tok: ["Constructs","a","BERT","Example",".","Args",":","hyp",":","Hypothesis","text",".","ref",":","Candidate","customization","variants","divided","by","'",";","'","target",":","if","infer==False",",","string","of","labels","(","each","label","is","1-based","index","of","correct","candidate",")","or","0.","if","infer==True",",","it","can","be","None","or","string","of","labels","(","each","label","is","1-based","index","of","some","candidate",")",".","In","inference","this","can","be","used","to","get","corresponding","fragments","to","fragment_indices",".","span_info",":","string","of","format","``","CUSTOM","6","20",";","CUSTOM","40","51","''",",","number","of","parts","corresponds","to","number","of","targets",".","Can","be","empty","if","target","is","0",".","If","infer==False",",","numbers","are","correct","start","and","end","(","exclusive",")","positions","of","the","corresponding","target","candidate","in","the","text",".","If","infer==True",",","numbers","are","EXPECTED","positions","in","the","text",".","In","inference","this","can","be","used","to","get","corresponding","fragments","to","fragment_indices",".","infer",":","inference","mode","Returns",":","BertExample",",","or","None","if","the","conversion","from","text","to","tags","was","infeasible","Example","(","infer=False",")",":","hyp",":","``","a","s","t","r","o","n","o","m","e","r","s","_","d","i","d","i","e","_","s","o","m","o","n","_","a","n","d","_","t","r","i","s","t","i","a","n","_","g","l","l","o","''","ref",":","``","d","i","d","i","e","r","_","s","a","u","m","o","n",";","a","s","t","r","o","n","o","m","i","e",";","t","r","i","s","t","a","n","_","g","u","i","l","l","o","t",";","t","r","i","s","t","e","s","s","e",";","m","o","n","a","d","e",";","c","h","r","i","s","t","i","a","n",";","a","s","t","r","o","n","o","m","e","r",";","s","o","l","o","m","o","n",";","d","i","d","i","d","i","d","i","d","i",";","m","e","r","c","y","''","target",":","``","1","3","''","span_info",":","``","CUSTOM","12","23",";","CUSTOM","28","41","''"]
code: def build_bert_example(
      self,
      hyp: str,
      ref: str,
      target: Optional[str] = None,
      span_info: Optional[str] = None,
      infer: bool = False,
  ) -> Optional[BertExample]:
      """Constructs a BERT Example.
  
      Args:
          hyp: Hypothesis text.
          ref: Candidate customization variants divided by ';'
          target:
              if infer==False, string of labels (each label is 1-based index of correct candidate) or 0.
              if infer==True, it can be None or string of labels (each label is 1-based index of some candidate). In inference this can be used to get corresponding fragments to fragment_indices.
          span_info:
              string of format "CUSTOM 6 20;CUSTOM 40 51", number of parts corresponds to number of targets. Can be empty if target is 0.
              If infer==False, numbers are correct start and end(exclusive) positions of the corresponding target candidate in the text.
              If infer==True, numbers are EXPECTED positions in the text. In inference this can be used to get corresponding fragments to fragment_indices.
          infer: inference mode
      Returns:
          BertExample, or None if the conversion from text to tags was infeasible
  
      Example (infer=False):
          hyp: "a s t r o n o m e r s _ d i d i e _ s o m o n _ a n d _ t r i s t i a n _ g l l o"
          ref: "d i d i e r _ s a u m o n;a s t r o n o m i e;t r i s t a n _ g u i l l o t;t r i s t e s s e;m o n a d e;c h r i s t i a n;a s t r o n o m e r;s o l o m o n;d i d i d i d i d i;m e r c y"
          target: "1 3"
          span_info: "CUSTOM 12 23;CUSTOM 28 41"
      """
      if not ref.count(";") == 9:
          raise ValueError("Expect 10 candidates: " + ref)
  
      span_info_parts = []
      targets = []
  
      if len(target) > 0 and target != "0":
          span_info_parts = span_info.split(";")
          targets = list(map(int, target.split(" ")))
          if len(span_info_parts) != len(targets):
              raise ValueError(
                  "len(span_info_parts)="
                  + str(len(span_info_parts))
                  + " is different from len(target_parts)="
                  + str(len(targets))
              )
  
      tags = [0 for _ in hyp.split()]
      if not infer:
          for p, t in zip(span_info_parts, targets):
              c, start, end = p.split(" ")
              start = int(start)
              end = int(end)
              tags[start:end] = [t for i in range(end - start)]
  
      # get input features for characters
      (
          input_ids,
          input_mask,
          segment_ids,
          labels_mask,
          labels,
          _,
          _,
      ) = self._get_input_features(hyp=hyp, ref=ref, tags=tags)
  
      # get input features for words
      hyp_with_words = hyp.replace(" ", "").replace("_", " ")
      ref_with_words = ref.replace(" ", "").replace("_", " ")
      (
          input_ids_for_subwords,
          input_mask_for_subwords,
          segment_ids_for_subwords,
          _,
          _,
          _,
          _,
      ) = self._get_input_features(
          hyp=hyp_with_words, ref=ref_with_words, tags=None
      )
  
      # used in forward to concatenate subword embeddings to character embeddings
      character_pos_to_subword_pos = self._map_characters_to_subwords(
          input_ids, input_ids_for_subwords
      )
  
      fragment_indices = []
      if infer:
          # used in inference to take argmax over whole fragments instead of separate characters to get more consistent predictions
          fragment_indices = self._get_fragment_indices(
              hyp, targets, span_info_parts
          )
  
      spans = []
      if not infer:
          # during training spans are used in validation step to calculate accuracy on whole custom phrases instead of separate characters
          spans = self._get_spans(span_info_parts)
  
      if (
          len(input_ids) > self._max_seq_length
          or len(spans) > self._max_spans_length
      ):
          print(
              "Max len exceeded: len(input_ids)=",
              len(input_ids),
              "; _max_seq_length=",
              self._max_seq_length,
              "; len(spans)=",
              len(spans),
              "; _max_spans_length=",
              self._max_spans_length,
          )
          return None
  
      example = BertExample(
          input_ids=input_ids,
          input_mask=input_mask,
          segment_ids=segment_ids,
          input_ids_for_subwords=input_ids_for_subwords,
          input_mask_for_subwords=input_mask_for_subwords,
          segment_ids_for_subwords=segment_ids_for_subwords,
          character_pos_to_subword_pos=character_pos_to_subword_pos,
          fragment_indices=fragment_indices,
          labels_mask=labels_mask,
          labels=labels,
          spans=spans,
          default_label=self._default_label,
      )
      return example
  
file_path: NeMo/nemo/collections/nlp/data/spellchecking_asr_customization/bert_example.py
imports: import logging
  from collections import OrderedDict
  from os import path
  from typing import Dict, List, Optional, Tuple, Union
  from transformers import PreTrainedTokenizerBase
label_desc: Use image node_id 2 for calling the BertExampleBuilder obj's underlying member method code with example usage: obj.build_bert_example(hyp, ref, target, span_info, infer) and returns: example, None