class_docstr: ASR model wrapper to perform alignment building.
  Functionality is limited to the components needed to build an alignment.
docstr: Transforms alignment information from token level to word level.
  
  Used when self._model.tokenizer is present.
docstr_tok: ["Transforms","alignment","information","from","token","level","to","word","level",".","Used","when","self._model.tokenizer","is","present","."]
code: def _process_tokens_to_words(
      self,
      tokens: List[str],
      token_begin: List[int],
      token_len: List[int],
      token_prob: List[float],
      words: List[str],
  ) -> Tuple[List[str], List[int], List[int], List[float]]:
      """Transforms alignment information from token level to word level.
  
      Used when self._model.tokenizer is present.
      """
      # suppose that there are no whitespaces
      assert len(self._model.tokenizer.text_to_tokens(words[0])) == len(
          self._model.tokenizer.text_to_tokens(words[0] + " ")
      )
      word_begin, word_len, word_prob = [], [], []
      token_len_nonzero = [(t_l if t_l > 0 else 1) for t_l in token_len]
      i = 0
      for word in words:
          loc_tokens = self._model.tokenizer.text_to_tokens(word)
          step = len(loc_tokens)
          # we assume that an empty word consists of only one token
          # drop current token
          if step == 0:
              token_begin[i + 1] = token_begin[i]
              token_len[i + 1] += token_len[i]
              token_len_nonzero[i + 1] += token_len_nonzero[i]
              del (
                  tokens[i],
                  token_begin[i],
                  token_len[i],
                  token_len_nonzero[i],
                  token_prob[i],
              )
              continue
          # fix <unk> tokenization
          if step == 2 and loc_tokens[-1] == "??":
              step -= 1
          j = i + step
          word_begin.append(token_begin[i])
          word_len.append(sum(token_len[i:j]))
          denominator = sum(token_len_nonzero[i:j])
          word_prob.append(
              sum(
                  token_prob[k] * token_len_nonzero[k]
                  for k in range(i, j)
              )
              / denominator
          )
          i = j
      return words, word_begin, word_len, word_prob
  
imports: import copy
  import os
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  import torch
  from omegaconf import DictConfig, OmegaConf, open_dict
  from tqdm.auto import tqdm
  from nemo.collections.asr.data.audio_to_ctm_dataset import FrameCtmUnit
  from nemo.collections.asr.data.audio_to_text_dali import DALIOutputs
  from nemo.collections.asr.models.asr_model import ASRModel
  from nemo.utils import logging
label_desc: Use image node_id 9 for calling the AlignerWrapperModel obj's underlying member method code with example usage: obj._process_tokens_to_words(tokens, token_begin, token_len, token_prob, words) and returns: words,  word_begin,  word_len,  word_prob