code: def aggregate_to_full_tokens(
      attn, tokens, token_starts, token_ends, attention=True
  ):
      to_combine = []
      spacy_attn = []
      spacy_token_starts = []
      spacy_token_ends = []
      spacy_start = None
      for token, prob, start, end in zip(
          tokens, attn, token_starts, token_ends
      ):
          to_combine.append(prob)
          if spacy_start is None:
              spacy_start = start
          if token.endswith("</w>"):
              if attention:
                  spacy_attn.append(np.max(to_combine, 0))
              else:
                  spacy_attn.append(to_combine[-1])
              spacy_token_starts.append(spacy_start)
              spacy_token_ends.append(end)
              to_combine = []
              spacy_start = None
  
      if attention:
          spacy_attn = spacy_attn / sum(spacy_attn)
          key = "attention_weights"
      else:
          key = "explanation"
  
      return {
          key: spacy_attn,
          "token_starts": spacy_token_starts,
          "token_ends": spacy_token_ends,
      }
  
file_path: finetune/finetune/base_models/gpt/encoder.py
imports: import re
  import json
  import os
  import warnings
  import codecs
  import numpy
  from ftfy.fixes import uncurl_quotes
  import finetune
  from finetune.encoding.input_encoder import get_spacy, EncodedOutput, BaseEncoder, get_pairs
label_desc: Use image node_id 2 for calling a global function with example usage: aggregate_to_full_tokens(attn, tokens, token_starts, token_ends, attention) and returns: dict