class_docstr: Dataset Class 
      1. Performs Model-dependent (but Data-independent) operations (tokenization etc)
      2. This can allow the same model preprocessing for multiple datasources
      3. Users can configurate which labels to use for modelling 
          (e.g. intent classification, slot filling or both together etc)
class_docstr_tok: ["Dataset","Class","1",".","Performs","Model-dependent","(","but","Data-independent",")","operations","(","tokenization","etc",")","2",".","This","can","allow","the","same","model","preprocessing","for","multiple","datasources","3",".","Users","can","configurate","which","labels","to","use","for","modelling","(","e.g",".","intent","classification",",","slot","filling","or","both","together","etc",")"]
docstr: Tokenize the utterance
  
  Args:
      utterance: A string containing the utterance to be tokenized.
  
  Returns:
      bert_tokens: A list of tokens obtained by word-piece tokenization of the
          utterance.
      alignments: A dict mapping indices of characters corresponding to start
          and end positions of words (not subwords) to corresponding indices in
          bert_tokens list.
      inverse_alignments: A list of size equal to bert_tokens. Each element is a
          tuple containing the index of the starting and inclusive ending
          character of the word corresponding to the subword. This list is used
          during inference to map word-piece indices to spans in the original
          utterance.
docstr_tok: ["Tokenize","the","utterance","Args",":","utterance",":","A","string","containing","the","utterance","to","be","tokenized",".","Returns",":","bert_tokens",":","A","list","of","tokens","obtained","by","word-piece","tokenization","of","the","utterance",".","alignments",":","A","dict","mapping","indices","of","characters","corresponding","to","start","and","end","positions","of","words","(","not","subwords",")","to","corresponding","indices","in","bert_tokens","list",".","inverse_alignments",":","A","list","of","size","equal","to","bert_tokens",".","Each","element","is","a","tuple","containing","the","index","of","the","starting","and","inclusive","ending","character","of","the","word","corresponding","to","the","subword",".","This","list","is","used","during","inference","to","map","word-piece","indices","to","spans","in","the","original","utterance","."]
code: def _tokenize(self, utterance: str):
      """
      Tokenize the utterance
  
      Args:
          utterance: A string containing the utterance to be tokenized.
  
      Returns:
          bert_tokens: A list of tokens obtained by word-piece tokenization of the
              utterance.
          alignments: A dict mapping indices of characters corresponding to start
              and end positions of words (not subwords) to corresponding indices in
              bert_tokens list.
          inverse_alignments: A list of size equal to bert_tokens. Each element is a
              tuple containing the index of the starting and inclusive ending
              character of the word corresponding to the subword. This list is used
              during inference to map word-piece indices to spans in the original
              utterance.
      """
      # utterance = tokenization.convert_to_unicode(utterance)
  
      # After _naive_tokenize, spaces and punctuation marks are all retained, i.e.
      # direct concatenation of all the tokens in the sequence will be the
      # original string.
      tokens = DialogueSGDBERTDataset._naive_tokenize(utterance)
      # ['I', ' ', 'am', ' ', 'feeling', ' ', 'hungry', ' ', 'so', ' ', 'I', ' ', 'would', ' ', 'like', ' ', 'to', ' ', 'find', ' ', 'a', ' ', 'place', ' ', 'to', ' ', 'eat', '.']
      # Filter out empty tokens and obtain aligned character index for each token.
      alignments = {}
      char_index = 0
      bert_tokens = (
          []
      )  # ['I', 'am', 'feeling', 'hungry', 'so', 'I', 'would', 'like', 'to', 'find', 'a', 'place', 'to', 'eat', '.']
      # These lists store inverse alignments to be used during inference.
      bert_tokens_start_chars = []
      bert_tokens_end_chars = []
      for token in tokens:
          if token.strip():
              subwords = self.tokenizer.text_to_tokens(token)
              # Store the alignment for the index of starting character and the
              # inclusive ending character of the token.
              alignments[char_index] = len(bert_tokens)
              bert_tokens_start_chars.extend(
                  [char_index] * len(subwords)
              )
              bert_tokens.extend(subwords)
              # The inclusive ending character index corresponding to the word.
              inclusive_char_end = char_index + len(token) - 1
              alignments[inclusive_char_end] = len(bert_tokens) - 1
              bert_tokens_end_chars.extend(
                  [inclusive_char_end] * len(subwords)
              )
          char_index += len(token)
      inverse_alignments = list(
          zip(bert_tokens_start_chars, bert_tokens_end_chars)
      )
      return bert_tokens, alignments, inverse_alignments
  
file_path: NeMo/nemo/collections/nlp/data/dialogue/dataset/dialogue_sgd_bert_dataset.py
imports: import os
  import re
  from typing import List
  import numpy
  from nemo.collections.nlp.data.dialogue.dataset.dialogue_dataset import DialogueDataset
  from nemo.collections.nlp.data.dialogue.input_example.sgd_input_example import SGDInputExample
label_desc: Use image node_id 5 for calling the DialogueSGDBERTDataset obj's underlying member method code with example usage: obj._tokenize(utterance) and returns: bert_tokens,  alignments,  inverse_alignments