class_docstr: Confidence Mixin class.
  
  It is responsible for confidence estimation method initialization and high-level confidence score calculation.
class_docstr_tok: ["Confidence","Mixin","class",".","It","is","responsible","for","confidence","estimation","method","initialization","and","high-level","confidence","score","calculation","."]
docstr: Implementation of token confidence aggregation for subword-based models.
  
  **Note**: Only supports Sentencepiece based tokenizers !
  
  Args:
      words: List of words of a hypothesis.
      token_confidence: List of token-level confidence scores of a hypothesis.
      token_ids: List of token ids of a hypothesis.
  
  Returns:
      A list of word-level confidence scores.
docstr_tok: ["Implementation","of","token","confidence","aggregation","for","subword-based","models",".","*","*","Note","*","*",":","Only","supports","Sentencepiece","based","tokenizers","!","Args",":","words",":","List","of","words","of","a","hypothesis",".","token_confidence",":","List","of","token-level","confidence","scores","of","a","hypothesis",".","token_ids",":","List","of","token","ids","of","a","hypothesis",".","Returns",":","A","list","of","word-level","confidence","scores","."]
code: def _aggregate_token_confidence_subwords_sentencepiece(
      self,
      words: List[str],
      token_confidence: List[float],
      token_ids: List[int],
  ) -> List[float]:
      """Implementation of token confidence aggregation for subword-based models.
  
      **Note**: Only supports Sentencepiece based tokenizers !
  
      Args:
          words: List of words of a hypothesis.
          token_confidence: List of token-level confidence scores of a hypothesis.
          token_ids: List of token ids of a hypothesis.
  
      Returns:
          A list of word-level confidence scores.
      """
      word_confidence = []
      # run only if there are final words
      if len(words) > 0:
          j = 0
          prev_unk = False
          prev_underline = False
          for i, token_id in enumerate(token_ids):
              token = self.decode_ids_to_tokens([int(token_id)])[0]
              token_text = self.decode_tokens_to_str([int(token_id)])
              # treat `<unk>` as a separate word regardless of the next token
              # to match the result of `tokenizer.ids_to_text`
              if (token != token_text or prev_unk) and i > j:
                  # do not add confidence for `▁` if the current token starts with `▁`
                  # to match the result of `tokenizer.ids_to_text`
                  if not prev_underline:
                      word_confidence.append(
                          self._aggregate_confidence(
                              token_confidence[j:i]
                          )
                      )
                  j = i
              prev_unk = token == "<unk>"
              prev_underline = token == "▁"
          if not prev_underline:
              word_confidence.append(
                  self._aggregate_confidence(
                      token_confidence[j : len(token_ids)]
                  )
              )
      if len(words) != len(word_confidence):
          raise RuntimeError(
              f"""Something went wrong with word-level confidence aggregation.\n
              Please check these values for debugging:\n
              len(words): {len(words)},\n
              len(word_confidence): {len(word_confidence)},\n
              recognized text: `{' '.join(words)}`"""
          )
      return word_confidence
  
imports: import math
  from abc import ABC, abstractmethod
  from dataclasses import dataclass, field
  from functools import partial
  from typing import List, Optional
  import torch
  from omegaconf import DictConfig, OmegaConf
  from nemo.collections.asr.parts.utils.rnnt_utils import Hypothesis
  from nemo.utils import logging
label_desc: Use image node_id 5 for calling the ConfidenceMixin obj's underlying member method code with example usage: obj._aggregate_token_confidence_subwords_sentencepiece(words, token_confidence, token_ids) and returns: word_confidence