class_docstr: Builder class for BertExample objects.
class_docstr_tok: ["Builder","class","for","BertExample","objects","."]
docstr: Maps each single character to the position of its corresponding subword.
  
  Args:
      input_ids: List of character token ids.
      input_ids_for_subwords: List of subword token ids.
  Returns:
      List of subword positions in input_ids_for_subwords. Its length is equal to len(input_ids)
  
  Example:
      input_ids: [101, 1037, 1055, 1056, 1054, 1051, 1050, ..., 1051, 102, 1040, ..., 1050, 102, 1037, ..., 1041, 102, ..., 102]
      input_ids_for_subwords: [101, 26357, 2106, 2666, 2061, 8202, 1998, 13012, 16643, 2319, 1043, 7174, 102, 2106, 3771, 7842, 2819, 2239, 102, ..., 102]
      result: [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, ... , 45, 46, 46, 46, 46, 46, 47]
docstr_tok: ["Maps","each","single","character","to","the","position","of","its","corresponding","subword",".","Args",":","input_ids",":","List","of","character","token","ids",".","input_ids_for_subwords",":","List","of","subword","token","ids",".","Returns",":","List","of","subword","positions","in","input_ids_for_subwords",".","Its","length","is","equal","to","len","(","input_ids",")","Example",":","input_ids",":","[","101",",","1037",",","1055",",","1056",",","1054",",","1051",",","1050",",","...",",","1051",",","102",",","1040",",","...",",","1050",",","102",",","1037",",","...",",","1041",",","102",",","...",",","102","]","input_ids_for_subwords",":","[","101",",","26357",",","2106",",","2666",",","2061",",","8202",",","1998",",","13012",",","16643",",","2319",",","1043",",","7174",",","102",",","2106",",","3771",",","7842",",","2819",",","2239",",","102",",","...",",","102","]","result",":","[","0",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","1",",","2",",","2",",","2",",","3",",","3",",","3",",","4",",","4",",","5",",","5",",","5",",","...",",","45",",","46",",","46",",","46",",","46",",","46",",","47","]"]
code: def _map_characters_to_subwords(
      self, input_ids: List[int], input_ids_for_subwords: List[int]
  ) -> List[int]:
      """Maps each single character to the position of its corresponding subword.
  
      Args:
          input_ids: List of character token ids.
          input_ids_for_subwords: List of subword token ids.
      Returns:
          List of subword positions in input_ids_for_subwords. Its length is equal to len(input_ids)
  
      Example:
          input_ids: [101, 1037, 1055, 1056, 1054, 1051, 1050, ..., 1051, 102, 1040, ..., 1050, 102, 1037, ..., 1041, 102, ..., 102]
          input_ids_for_subwords: [101, 26357, 2106, 2666, 2061, 8202, 1998, 13012, 16643, 2319, 1043, 7174, 102, 2106, 3771, 7842, 2819, 2239, 102, ..., 102]
          result: [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, ... , 45, 46, 46, 46, 46, 46, 47]
      """
      character_pos_to_subword_pos = [0 for _ in input_ids]
  
      ## '[CLS]', 'a', 's', 't', 'r', 'o', 'n', 'o', 'm', 'e', 'r', 's', '_', 'd', 'i', ..., 'l', 'o', '[SEP]', 'd', 'i', 'd', 'i', 'e', 'r', '_', 's', 'a', 'u', 'm', 'o', 'n', ..., '[SEP]'
      tokens = self._tokenizer.convert_ids_to_tokens(input_ids)
      ## '[CLS]', 'astronomers', 'did', '##ie', 'so', '##mon', 'and', 'tri', '##sti', '##an', 'g', '##llo', '[SEP]', 'did', '##ier', 'sa', '##um', '##on', '[SEP]', 'astro', '##no', '##mie', '[SEP]', 'tristan', 'gui', '##llo', '##t', '[SEP]', ..., '[SEP]', 'mercy', '[SEP]']
      tokens_for_subwords = self._tokenizer.convert_ids_to_tokens(
          input_ids_for_subwords
      )
      j = 0  # index for tokens_for_subwords
      j_offset = 0  # current letter index within subword
      for i in range(len(tokens)):
          character = tokens[i]
          subword = tokens_for_subwords[j]
          if character == "[CLS]" and subword == "[CLS]":
              character_pos_to_subword_pos[i] = j
              j += 1
              continue
          if character == "[SEP]" and subword == "[SEP]":
              character_pos_to_subword_pos[i] = j
              j += 1
              continue
          if (
              character == "[CLS]"
              or character == "[SEP]"
              or subword == "[CLS]"
              or subword == "[SEP]"
          ):
              raise IndexError(
                  "character["
                  + str(i)
                  + "]="
                  + character
                  + "; subword["
                  + str(j)
                  + ";="
                  + subword
                  + "subwords="
                  + str(tokens_for_subwords)
              )
          # At this point we expect that
          #    subword either 1) is a normal first token of a word or 2) starts with "##" (not first word token)
          #    character either 1) is a normal character or 2) is a space character "_"
          if character == "_":
              character_pos_to_subword_pos[i] = (
                  j - 1
              )  # space is assigned to previous subtoken
              continue
          if j_offset < len(subword):
              if character == subword[j_offset]:
                  character_pos_to_subword_pos[i] = j
                  j_offset += 1
              else:
                  raise IndexError(
                      "character mismatch:"
                      + "i="
                      + str(i)
                      + "j="
                      + str(j)
                      + "j_offset="
                      + str(j_offset)
                      + "; len(tokens)="
                      + str(len(tokens))
                      + "; len(subwords)="
                      + str(len(tokens_for_subwords))
                  )
          # if subword is finished, increase j
          if j_offset >= len(subword):
              j += 1
              j_offset = 0
              if j >= len(tokens_for_subwords):
                  break
              if tokens_for_subwords[j].startswith("##"):
                  j_offset = 2
      # check that all subword tokens are processed
      if j < len(tokens_for_subwords):
          raise IndexError(
              "j="
              + str(j)
              + "; len(tokens)="
              + str(len(tokens))
              + "; len(subwords)="
              + str(len(tokens_for_subwords))
          )
      return character_pos_to_subword_pos
  
file_path: NeMo/nemo/collections/nlp/data/spellchecking_asr_customization/bert_example.py
imports: import logging
  from collections import OrderedDict
  from os import path
  from typing import Dict, List, Optional, Tuple, Union
  from transformers import PreTrainedTokenizerBase
label_desc: Use image node_id 5 for calling the BertExampleBuilder obj's underlying member method code with example usage: obj._map_characters_to_subwords(input_ids, input_ids_for_subwords) and returns: character_pos_to_subword_pos