code: def decode_predictions(
      vocab,
      mwt_dict,
      orig_text,
      all_raw,
      all_preds,
      no_ssplit,
      skip_newline,
      use_la_ittb_shorthand,
  ):
      """
      Decode the predictions into a document of words
  
      Once everything is fed through the tokenizer model, it's time to decode the predictions
      into actual tokens and sentences that the rest of the pipeline uses
      """
      offset = 0
      oov_count = 0
      doc = []
  
      text = (
          SPACE_RE.sub(" ", orig_text)
          if orig_text is not None
          else None
      )
      char_offset = 0
  
      if vocab is not None:
          UNK_ID = vocab.unit2id("<UNK>")
  
      for raw, pred in zip(all_raw, all_preds):
          current_tok = ""
          current_sent = []
  
          for t, p in zip(raw, pred):
              if t == "<PAD>":
                  break
              # hack la_ittb
              if use_la_ittb_shorthand and t in (":", ";"):
                  p = 2
              offset += 1
              if vocab is not None and vocab.unit2id(t) == UNK_ID:
                  oov_count += 1
  
              current_tok += t
              if p >= 1:
                  if vocab is not None:
                      tok = vocab.normalize_token(current_tok)
                  else:
                      tok = current_tok
                  assert "\t" not in tok, tok
                  if len(tok) <= 0:
                      current_tok = ""
                      continue
                  if orig_text is not None:
                      st = -1
                      tok_len = 0
                      for part in SPACE_SPLIT_RE.split(current_tok):
                          if len(part) == 0:
                              continue
                          if skip_newline:
                              part_pattern = re.compile(
                                  r"\s*".join(
                                      re.escape(c) for c in part
                                  )
                              )
                              match = part_pattern.search(
                                  text, char_offset
                              )
                              st0 = match.start(0) - char_offset
                              partlen = match.end(0) - match.start(0)
                              lstripped = match.group(0).lstrip()
                          else:
                              try:
                                  st0 = (
                                      text.index(part, char_offset)
                                      - char_offset
                                  )
                              except ValueError as e:
                                  sub_start = max(0, char_offset - 20)
                                  sub_end = min(
                                      len(text), char_offset + 20
                                  )
                                  sub = text[sub_start:sub_end]
                                  raise ValueError(
                                      "Could not find |%s| starting from char_offset %d.  Surrounding text: |%s|"
                                      % (part, char_offset, sub)
                                  ) from e
                              partlen = len(part)
                              lstripped = part.lstrip()
                          if st < 0:
                              st = (
                                  char_offset
                                  + st0
                                  + (partlen - len(lstripped))
                              )
                          char_offset += st0 + partlen
                      position_info = (st, char_offset)
                  else:
                      position_info = None
                  current_sent.append((tok, p, position_info))
                  current_tok = ""
                  if (p == 2 or p == 4) and not no_ssplit:
                      doc.append(
                          process_sentence(current_sent, mwt_dict)
                      )
                      current_sent = []
  
          if len(current_tok) > 0:
              raise ValueError(
                  "Finished processing tokens, but there is still text left!"
              )
          if len(current_sent):
              doc.append(process_sentence(current_sent, mwt_dict))
  
      return oov_count, offset, doc
  
imports: from collections import Counter
  from copy import copy
  import json
  import numpy
  import re
  import logging
  import os
  from torch.utils.data import DataLoader
  import stanza.utils.default_paths
  from stanza.models.common.utils import ud_scores, harmonic_mean
  from stanza.models.common.doc import Document
  from stanza.utils.conll import CoNLL
  from stanza.models.common.doc import *
  from stanza.models.tokenization.data import SortedDataset
label_desc: Use image node_id 12 for calling a global function with example usage: decode_predictions(vocab, mwt_dict, orig_text, all_raw, all_preds, no_ssplit, skip_newline, use_la_ittb_shorthand) and returns: oov_count,  offset,  doc