code: def get_utt_obj(
      text,
      model,
      separator,
      T,
      audio_filepath,
      utt_id,
  ):
      """
      Function to create an Utterance object and add all necessary information to it except
          for timings of the segments / words / tokens according to the alignment - that will
          be done later in a different function, after the alignment is done.
  
          The Utterance object has a list segments_and_tokens which contains Segment objects and
          Token objects (for blank tokens in between segments).
          Within the Segment objects, there is a list words_and_tokens which contains Word objects and
          Token objects (for blank tokens in between words).
          Within the Word objects, there is a list tokens tokens which contains Token objects for
          blank and non-blank tokens.
          We will be building up these lists in this function. This data structure will then be useful for
          generating the various output files that we wish to save.
      """
  
      if (
          not separator
      ):  # if separator is not defined - treat the whole text as one segment
          segments = [text]
      else:
          segments = text.split(separator)
  
      # remove any spaces at start and end of segments
      segments = [seg.strip() for seg in segments]
      # remove any empty segments
      segments = [seg for seg in segments if len(seg) > 0]
  
      utt = Utterance(
          text=text,
          audio_filepath=audio_filepath,
          utt_id=utt_id,
      )
  
      # build up lists: token_ids_with_blanks, segments_and_tokens.
      # The code for these is different depending on whether we use char-based tokens or not
      if hasattr(model, "tokenizer"):
          if hasattr(model, "blank_id"):
              BLANK_ID = model.blank_id
          else:
              BLANK_ID = len(model.tokenizer.vocab)  # TODO: check
  
          utt.token_ids_with_blanks = [BLANK_ID]
  
          # check for text being 0 length
          if len(text) == 0:
              return utt
  
          # check for # tokens + token repetitions being > T
          all_tokens = model.tokenizer.text_to_ids(text)
          n_token_repetitions = 0
          for i_tok in range(1, len(all_tokens)):
              if all_tokens[i_tok] == all_tokens[i_tok - 1]:
                  n_token_repetitions += 1
  
          if len(all_tokens) + n_token_repetitions > T:
              logging.info(
                  f"Utterance {utt_id} has too many tokens compared to the audio file duration."
                  " Will not generate output alignment files for this utterance."
              )
              return utt
  
          # build up data structures containing segments/words/tokens
          utt.segments_and_tokens.append(
              Token(
                  text=BLANK_TOKEN,
                  text_cased=BLANK_TOKEN,
                  s_start=0,
                  s_end=0,
              )
          )
  
          segment_s_pointer = 1  # first segment will start at s=1 because s=0 is a blank
          word_s_pointer = (
              1  # first word will start at s=1 because s=0 is a blank
          )
  
          for segment in segments:
              # add the segment to segment_info and increment the segment_s_pointer
              segment_tokens = model.tokenizer.text_to_tokens(segment)
              utt.segments_and_tokens.append(
                  Segment(
                      text=segment,
                      s_start=segment_s_pointer,
                      # segment_tokens do not contain blanks => need to muliply by 2
                      # s_end needs to be the index of the final token (including blanks) of the current segment:
                      # segment_s_pointer + len(segment_tokens) * 2 is the index of the first token of the next segment =>
                      # => need to subtract 2
                      s_end=segment_s_pointer
                      + len(segment_tokens) * 2
                      - 2,
                  )
              )
              segment_s_pointer += (
                  len(segment_tokens) * 2
              )  # multiply by 2 to account for blanks (which are not present in segment_tokens)
  
              words = segment.split(
                  " "
              )  # we define words to be space-separated sub-strings
              for word_i, word in enumerate(words):
                  word_tokens = model.tokenizer.text_to_tokens(word)
                  word_token_ids = model.tokenizer.text_to_ids(word)
                  word_tokens_cased = restore_token_case(
                      word, word_tokens
                  )
  
                  # add the word to word_info and increment the word_s_pointer
                  utt.segments_and_tokens[-1].words_and_tokens.append(
                      # word_tokens do not contain blanks => need to muliply by 2
                      # s_end needs to be the index of the final token (including blanks) of the current word:
                      # word_s_pointer + len(word_tokens) * 2 is the index of the first token of the next word =>
                      # => need to subtract 2
                      Word(
                          text=word,
                          s_start=word_s_pointer,
                          s_end=word_s_pointer
                          + len(word_tokens) * 2
                          - 2,
                      )
                  )
                  word_s_pointer += (
                      len(word_tokens) * 2
                  )  # multiply by 2 to account for blanks (which are not present in word_tokens)
  
                  for token_i, (
                      token,
                      token_id,
                      token_cased,
                  ) in enumerate(
                      zip(
                          word_tokens, word_token_ids, word_tokens_cased
                      )
                  ):
                      # add the text tokens and the blanks in between them
                      # to our token-based variables
                      utt.token_ids_with_blanks.extend(
                          [token_id, BLANK_ID]
                      )
                      # adding Token object for non-blank token
                      utt.segments_and_tokens[-1].words_and_tokens[
                          -1
                      ].tokens.append(
                          Token(
                              text=token,
                              text_cased=token_cased,
                              # utt.token_ids_with_blanks has the form [...., <this non-blank token>, <blank>] =>
                              # => if do len(utt.token_ids_with_blanks) - 1 you get the index of the final <blank>
                              # => we want to do len(utt.token_ids_with_blanks) - 2 to get the index of <this non-blank token>
                              s_start=len(utt.token_ids_with_blanks)
                              - 2,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 2,
                          )
                      )
  
                      # adding Token object for blank tokens in between the tokens of the word
                      # (ie do not add another blank if you have reached the end)
                      if token_i < len(word_tokens) - 1:
                          utt.segments_and_tokens[-1].words_and_tokens[
                              -1
                          ].tokens.append(
                              Token(
                                  text=BLANK_TOKEN,
                                  text_cased=BLANK_TOKEN,
                                  # utt.token_ids_with_blanks has the form [...., <this blank token>] =>
                                  # => if do len(utt.token_ids_with_blanks) -1 you get the index of this <blank>
                                  s_start=len(utt.token_ids_with_blanks)
                                  - 1,
                                  # s_end is same as s_start since the token only occupies one element in the list
                                  s_end=len(utt.token_ids_with_blanks)
                                  - 1,
                              )
                          )
  
                  # add a Token object for blanks in between words in this segment
                  # (but only *in between* - do not add the token if it is after the final word)
                  if word_i < len(words) - 1:
                      utt.segments_and_tokens[
                          -1
                      ].words_and_tokens.append(
                          Token(
                              text=BLANK_TOKEN,
                              text_cased=BLANK_TOKEN,
                              # utt.token_ids_with_blanks has the form [...., <this blank token>] =>
                              # => if do len(utt.token_ids_with_blanks) -1 you get the index of this <blank>
                              s_start=len(utt.token_ids_with_blanks)
                              - 1,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 1,
                          )
                      )
  
              # add the blank token in between segments/after the final segment
              utt.segments_and_tokens.append(
                  Token(
                      text=BLANK_TOKEN,
                      text_cased=BLANK_TOKEN,
                      # utt.token_ids_with_blanks has the form [...., <this blank token>] =>
                      # => if do len(utt.token_ids_with_blanks) -1 you get the index of this <blank>
                      s_start=len(utt.token_ids_with_blanks) - 1,
                      # s_end is same as s_start since the token only occupies one element in the list
                      s_end=len(utt.token_ids_with_blanks) - 1,
                  )
              )
  
          return utt
  
      elif hasattr(
          model.decoder, "vocabulary"
      ):  # i.e. tokenization is simply character-based
          BLANK_ID = len(
              model.decoder.vocabulary
          )  # TODO: check this is correct
          SPACE_ID = model.decoder.vocabulary.index(" ")
  
          utt.token_ids_with_blanks = [BLANK_ID]
  
          # check for text being 0 length
          if len(text) == 0:
              return utt
  
          # check for # tokens + token repetitions being > T
          all_tokens = get_char_tokens(text, model)
          n_token_repetitions = 0
          for i_tok in range(1, len(all_tokens)):
              if all_tokens[i_tok] == all_tokens[i_tok - 1]:
                  n_token_repetitions += 1
  
          if len(all_tokens) + n_token_repetitions > T:
              logging.info(
                  f"Utterance {utt_id} has too many tokens compared to the audio file duration."
                  " Will not generate output alignment files for this utterance."
              )
              return utt
  
          # build up data structures containing segments/words/tokens
          utt.segments_and_tokens.append(
              Token(
                  text=BLANK_TOKEN,
                  text_cased=BLANK_TOKEN,
                  s_start=0,
                  s_end=0,
              )
          )
  
          segment_s_pointer = 1  # first segment will start at s=1 because s=0 is a blank
          word_s_pointer = (
              1  # first word will start at s=1 because s=0 is a blank
          )
  
          for i_segment, segment in enumerate(segments):
              # add the segment to segment_info and increment the segment_s_pointer
              segment_tokens = get_char_tokens(segment, model)
              utt.segments_and_tokens.append(
                  Segment(
                      text=segment,
                      s_start=segment_s_pointer,
                      # segment_tokens do not contain blanks => need to muliply by 2
                      # s_end needs to be the index of the final token (including blanks) of the current segment:
                      # segment_s_pointer + len(segment_tokens) * 2 is the index of the first token of the next segment =>
                      # => need to subtract 2
                      s_end=segment_s_pointer
                      + len(segment_tokens) * 2
                      - 2,
                  )
              )
  
              # for correct calculation: multiply len(segment_tokens) by 2 to account for blanks (which are not present in segment_tokens)
              # and + 2 to account for [<token for space in between segments>, <blank token after that space token>]
              segment_s_pointer += len(segment_tokens) * 2 + 2
  
              words = segment.split(
                  " "
              )  # we define words to be space-separated substrings
              for i_word, word in enumerate(words):
                  # convert string to list of characters
                  word_tokens = list(word)
                  # convert list of characters to list of their ids in the vocabulary
                  word_token_ids = get_char_tokens(word, model)
  
                  # add the word to word_info and increment the word_s_pointer
                  utt.segments_and_tokens[-1].words_and_tokens.append(
                      # note for s_end:
                      # word_tokens do not contain blanks => need to muliply by 2
                      # s_end needs to be the index of the final token (including blanks) of the current word:
                      # word_s_pointer + len(word_tokens) * 2 is the index of the first token of the next word =>
                      # => need to subtract 2
                      Word(
                          text=word,
                          s_start=word_s_pointer,
                          s_end=word_s_pointer
                          + len(word_tokens) * 2
                          - 2,
                      )
                  )
  
                  # for correct calculation: multiply len(word_tokens) by 2 to account for blanks (which are not present in word_tokens)
                  # and + 2 to account for [<token for space in between words>, <blank token after that space token>]
                  word_s_pointer += len(word_tokens) * 2 + 2
  
                  for token_i, (token, token_id) in enumerate(
                      zip(word_tokens, word_token_ids)
                  ):
                      # add the text tokens and the blanks in between them
                      # to our token-based variables
                      utt.token_ids_with_blanks.extend([token_id])
                      utt.segments_and_tokens[-1].words_and_tokens[
                          -1
                      ].tokens.append(
                          Token(
                              text=token,
                              text_cased=token,
                              # utt.token_ids_with_blanks has the form [..., <this non-blank token>]
                              # => do len(utt.token_ids_with_blanks) - 1 to get the index of this non-blank token
                              s_start=len(utt.token_ids_with_blanks)
                              - 1,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 1,
                          )
                      )
  
                      if (
                          token_i < len(word_tokens) - 1
                      ):  # only add blank tokens that are in the middle of words
                          utt.token_ids_with_blanks.extend([BLANK_ID])
                          utt.segments_and_tokens[-1].words_and_tokens[
                              -1
                          ].tokens.append(
                              Token(
                                  text=BLANK_TOKEN,
                                  text_cased=BLANK_TOKEN,
                                  # utt.token_ids_with_blanks has the form [..., <this blank token>]
                                  # => do len(utt.token_ids_with_blanks) - 1 to get the index of this blank token
                                  s_start=len(utt.token_ids_with_blanks)
                                  - 1,
                                  # s_end is same as s_start since the token only occupies one element in the list
                                  s_end=len(utt.token_ids_with_blanks)
                                  - 1,
                              )
                          )
  
                  # add space token (and the blanks around it) unless this is the final word in a segment
                  if i_word < len(words) - 1:
                      utt.token_ids_with_blanks.extend(
                          [BLANK_ID, SPACE_ID, BLANK_ID]
                      )
                      utt.segments_and_tokens[
                          -1
                      ].words_and_tokens.append(
                          Token(
                              text=BLANK_TOKEN,
                              text_cased=BLANK_TOKEN,
                              # utt.token_ids_with_blanks has the form
                              # [..., <final token of previous word>, <blank token>, <space token>, <blank token>]
                              # => do len(utt.token_ids_with_blanks) - 3 to get the index of the blank token before the space token
                              s_start=len(utt.token_ids_with_blanks)
                              - 3,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 3,
                          )
                      )
                      utt.segments_and_tokens[
                          -1
                      ].words_and_tokens.append(
                          Token(
                              text=SPACE_TOKEN,
                              text_cased=SPACE_TOKEN,
                              # utt.token_ids_with_blanks has the form
                              # [..., <final token of previous word>, <blank token>, <space token>, <blank token>]
                              # => do len(utt.token_ids_with_blanks) - 2 to get the index of the space token
                              s_start=len(utt.token_ids_with_blanks)
                              - 2,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 2,
                          )
                      )
                      utt.segments_and_tokens[
                          -1
                      ].words_and_tokens.append(
                          Token(
                              text=BLANK_TOKEN,
                              text_cased=BLANK_TOKEN,
                              # utt.token_ids_with_blanks has the form
                              # [..., <final token of previous word>, <blank token>, <space token>, <blank token>]
                              # => do len(utt.token_ids_with_blanks) - 1 to get the index of the blank token after the space token
                              s_start=len(utt.token_ids_with_blanks)
                              - 1,
                              # s_end is same as s_start since the token only occupies one element in the list
                              s_end=len(utt.token_ids_with_blanks) - 1,
                          )
                      )
  
              # add a blank to the segment, and add a space after if this is not the final segment
              utt.token_ids_with_blanks.extend([BLANK_ID])
              utt.segments_and_tokens.append(
                  Token(
                      text=BLANK_TOKEN,
                      text_cased=BLANK_TOKEN,
                      # utt.token_ids_with_blanks has the form [..., <this blank token>]
                      # => do len(utt.token_ids_with_blanks) - 1 to get the index of this blank token
                      s_start=len(utt.token_ids_with_blanks) - 1,
                      # s_end is same as s_start since the token only occupies one element in the list
                      s_end=len(utt.token_ids_with_blanks) - 1,
                  )
              )
  
              if i_segment < len(segments) - 1:
                  utt.token_ids_with_blanks.extend([SPACE_ID, BLANK_ID])
                  utt.segments_and_tokens.append(
                      Token(
                          text=SPACE_TOKEN,
                          text_cased=SPACE_TOKEN,
                          # utt.token_ids_with_blanks has the form
                          # [..., <space token>, <blank token>]
                          # => do len(utt.token_ids_with_blanks) - 2 to get the index of the space token
                          s_start=len(utt.token_ids_with_blanks) - 2,
                          # s_end is same as s_start since the token only occupies one element in the list
                          s_end=len(utt.token_ids_with_blanks) - 2,
                      )
                  )
                  utt.segments_and_tokens.append(
                      Token(
                          text=BLANK_TOKEN,
                          text_cased=BLANK_TOKEN,
                          # utt.token_ids_with_blanks has the form
                          # [..., <space token>, <blank token>]
                          # => do len(utt.token_ids_with_blanks) - 1 to get the index of the blank token
                          s_start=len(utt.token_ids_with_blanks) - 1,
                          # s_end is same as s_start since the token only occupies one element in the list
                          s_end=len(utt.token_ids_with_blanks) - 1,
                      )
                  )
  
          return utt
  
      else:
          raise RuntimeError("Cannot get tokens of this model.")
  
imports: import json
  from dataclasses import dataclass, field
  from pathlib import Path
  from typing import List, Union
  import soundfile
  import torch
  from tqdm.auto import tqdm
  from utils.constants import BLANK_TOKEN, SPACE_TOKEN, V_NEGATIVE_NUM
  from nemo.utils import logging
label_desc: Use image node_id 9 for calling a global function with example usage: get_utt_obj(text, model, separator, T, audio_filepath, utt_id) and returns: utt, utt, utt, utt, utt, utt