code: def partition_tree(decoded_tokens, special_tokens):
      """Build a heriarchial clustering of tokens that align with sentence structure.
  
      Note that this is fast and heuristic right now.
      TODO: Build this using a real constituency parser.
      """
      token_groups = [
          TokenGroup([Token(t)], i)
          for i, t in enumerate(decoded_tokens)
      ]
      #     print(token_groups)
      M = len(decoded_tokens)
      new_index = M
      clustm = np.zeros((M - 1, 4))
      for i in range(len(token_groups) - 1):
          scores = [
              merge_score(
                  token_groups[i], token_groups[i + 1], special_tokens
              )
              for i in range(len(token_groups) - 1)
          ]
          #         print(scores)
          ind = np.argmax(scores)
  
          lind = token_groups[ind].index
          rind = token_groups[ind + 1].index
          clustm[new_index - M, 0] = token_groups[ind].index
          clustm[new_index - M, 1] = token_groups[ind + 1].index
          clustm[new_index - M, 2] = -scores[ind]
          clustm[new_index - M, 3] = (
              clustm[lind - M, 3] if lind >= M else 1
          ) + (clustm[rind - M, 3] if rind >= M else 1)
  
          token_groups[ind] = token_groups[ind] + token_groups[ind + 1]
          token_groups[ind].index = new_index
  
          # track balancing of openers/closers
          if (
              token_groups[ind][0].s in openers
              and token_groups[ind + 1][-1].s
              == openers[token_groups[ind][0].s]
          ):
              token_groups[ind][0].balanced = True
              token_groups[ind + 1][-1].balanced = True
  
          token_groups.pop(ind + 1)
          new_index += 1
  
      # negative means we should never split a group, so we add 10 to ensure these are very tight groups
      # (such as parts of the same word)
      clustm[:, 2] = clustm[:, 2] + 10
  
      return clustm
  
imports: import math
  import re
  import numpy
  from .._serializable import Deserializer, Serializer
  from ..utils import safe_isinstance
  from ..utils.transformers import SENTENCEPIECE_TOKENIZERS, getattr_silent, parse_prefix_suffix_for_tokenizer
  from ._masker import Masker
label_desc: Use image node_id 4 for calling a global function with example usage: partition_tree(decoded_tokens, special_tokens) and returns: clustm