code: def finetune_to_indico_explain(
      raw_texts, attn_weights, encoder, attention=True
  ):
      """
      Maps the attention weights or explain probability values one-to-one with the raw text tokens.
  
      :param raw_texts: A list of segmented text of the form list(list(str))
      :param attn_weights: An array of attention weights of shape [batch, seq_len]
      :param encoder: The encoder used in the model that output the attention weights
      :param attention: A boolean flag, true for inputting attention matrices, false for explain probabilities.
      :return: A list of dictionaries, each with the following keys:
      - 'attention_weights': A list of attention weights for each token in the text
      - 'token_starts': A list of the start tokens for each spacy token in the text
      - 'token_ends': A list of the end tokens for each spacy token in the text
  
      """
      spacy_outputs = []
  
      encoded_docs = encoder._encode(raw_texts)
  
      for doc_idx, (raw_text) in enumerate(raw_texts):
          tokens = encoded_docs.tokens[doc_idx]
          token_ends = encoded_docs.token_ends[doc_idx]
          token_starts = encoded_docs.token_ends[doc_idx]
          if attention:
              # offset of one to take into account the start token
              clf_token_idx = len(tokens) + 1
              # take the average values over the attention heads for the attention weights of the classify token
              attn = np.mean(attn_weights[doc_idx], axis=0)[
                  clf_token_idx
              ][
                  1:clf_token_idx
              ]  # [num_tokens]
          else:
              # There is no start token
              clf_token_idx = len(tokens)
              attn = attn_weights[doc_idx][:clf_token_idx]
          # map one-to-one with spacy tokenization
          spacy_output = aggregate_to_full_tokens(
              attn,
              tokens,
              token_starts,
              token_ends,
              attention=attention,
          )
          spacy_outputs.append(spacy_output)
      return spacy_outputs
  
file_path: finetune/finetune/base_models/gpt/encoder.py
imports: import re
  import json
  import os
  import warnings
  import codecs
  import numpy
  from ftfy.fixes import uncurl_quotes
  import finetune
  from finetune.encoding.input_encoder import get_spacy, EncodedOutput, BaseEncoder, get_pairs
label_desc: Use image node_id 3 for calling a global function with example usage: finetune_to_indico_explain(raw_texts, attn_weights, encoder, attention) and returns: spacy_outputs