code: def finetune_to_indico_sequence(
      raw_texts,
      subseqs,
      labels,
      probs=None,
      none_value=None,
      subtoken_predictions=False,
      associations=None,
      bio_tagging=False,
  ):
      """
      Maps from the labeled substring format into the 'indico' format. This is the exact inverse operation to
      :meth indico_to_finetune_sequence:.
  
      The indico format is as follows:
          Raw text for X,
          Labels as a list of dicts, with each dict in the form:
          {
              'start': <Character index of the start_token of the labeled sequence>,
              'end': <Character index of the end of the labeled sequence>,
              'label': <A categorical label (int or string) that represents the category of the subsequence,
              'text': <Optionally, a field with the subsequence contained between the start and end.
          }
  
      The Labeled substring, or finetune internal, format is as follows.
      Each item of the data is a list strings of the form:
          ["The quick brown", "fox", "jumped over the lazy", ...]
      With the corresponding labels:
          ["PAD", "animal", "PAD", ...]
  
      It is the :param none_value: that is used to populate the PAD labels.
      :param data: A list of segmented text of the form list(list(str))
      :param labels: Categorical labels for each sub-string in data.
      :param none_value: The none value used to encode the input format.
      :return: Texts, annoatations both in the 'indico' format.
      """
      annotations = []
      if not subtoken_predictions:
          spacy_docs = get_spacy().pipe(raw_texts)
      else:
          spacy_docs = [None] * len(raw_texts)
      loop_vals = zip(
          raw_texts,
          spacy_docs,
          subseqs,
          labels,
          probs or [None] * len(raw_texts),
      )
      for doc_idx, (
          raw_text,
          spacy_tokens,
          doc_seq,
          label_seq,
          prob_seq,
      ) in enumerate(loop_vals):
          if not subtoken_predictions:
              spacy_token_starts = np.asarray(
                  [token.idx for token in spacy_tokens]
              )
              spacy_token_ends = np.asarray(
                  [
                      token.idx + len(token.text)
                      for token in spacy_tokens
                  ]
              )
          doc_annotations = []
          annotation_ranges = set()
          raw_annotation_end = 0
          raw_annotation_start = 0
          subtoken_to_label_idx = []
          for i, (sub_str, raw_label, confidences) in enumerate(
              zip(doc_seq, label_seq, prob_seq or [None] * len(doc_seq))
          ):
              subtoken_to_label_idx.append(len(doc_annotations))
              if not isinstance(raw_label, tuple):
                  multilabel = False
                  label_list = [raw_label]
              else:
                  multilabel = True
                  label_list = raw_label
  
              for label_idx, label in enumerate(label_list):
                  stripped_text = sub_str.strip()
                  start_of_search = (
                      raw_annotation_start
                      if label_idx != 0
                      else raw_annotation_end
                  )
                  if subtoken_predictions:
                      raw_annotation_start = raw_text.find(
                          sub_str, start_of_search
                      )
                      raw_annotation_end = raw_annotation_start + len(
                          sub_str
                      )
                  else:
                      raw_annotation_start = raw_text.find(
                          stripped_text, start_of_search
                      )
                      raw_annotation_end = raw_annotation_start + len(
                          stripped_text
                      )
  
                  if raw_annotation_start == -1:
                      warnings.warn(
                          "Failed to find predicted sequence: {} in text".format(
                              sub_str
                          )
                      )
                      continue
  
                  extended_existing_label = False
                  for item in (
                      doc_annotations
                      if multilabel
                      else doc_annotations[-1:]
                  ):
                      # handle case where we extend existing annotation
                      if (
                          # same label
                          item["label"] == label
                          # and only separated by whitespace
                          and item["end"] <= raw_annotation_end
                          and not raw_text[
                              item["end"] : raw_annotation_start
                          ].strip()
                          # and not BIO tagging
                          and not bio_tagging
                      ):
                          item["end"] = raw_annotation_end
                          item["text"] = raw_text[
                              item["start"] : raw_annotation_end
                          ]
                          if (
                              "confidence" in item
                              and confidences is not None
                          ):
                              item["confidence"].append(confidences)
                          extended_existing_label = True
                          break
  
                  if extended_existing_label or label == none_value:
                      continue
  
                  annotation_start, annotation_end = (
                      int(raw_annotation_start),
                      int(raw_annotation_end),
                  )
  
                  annotation = {
                      "start": annotation_start,
                      "end": annotation_end,
                      "label": label,
                      "text": raw_text[annotation_start:annotation_end],
                  }
  
                  if annotation["text"].strip() != annotation["text"]:
                      annotation = strip_annotation_whitespace(
                          annotation
                      )
  
                  # if we don't want to allow subtoken predictions, adjust start and end to match
                  # the start and ends of the nearest full tokens
                  if not subtoken_predictions:
                      round_to_nearest_start_and_end(
                          annotation,
                          spacy_token_starts,
                          spacy_token_ends,
                          raw_text,
                      )
  
                  if confidences is not None:
                      annotation["confidence"] = [confidences]
  
                  if annotation["start"] >= annotation["end"]:
                      continue
  
                  # prevent duplicate annotation edge case
                  annotation_tuple = (
                      annotation["start"],
                      annotation["end"],
                      label,
                  )
                  if annotation_tuple not in annotation_ranges:
                      annotation_ranges.add(annotation_tuple)
                      doc_annotations.append(annotation)
  
          doc_annotations = sorted(
              [dict(items) for items in doc_annotations],
              key=lambda x: span(x),
          )
  
          for annotation in doc_annotations:
              _merge_confidences(annotation)
  
          annotations.append(doc_annotations)
      return raw_texts, annotations
  
file_path: finetune/finetune/encoding/sequence_encoder.py
imports: import warnings
  import numpy
  from finetune.encoding.input_encoder import get_spacy
label_desc: Use image node_id 4 for calling a global function with example usage: finetune_to_indico_sequence(raw_texts, subseqs, labels, probs, none_value, subtoken_predictions, associations, bio_tagging) and returns: raw_texts,  annotations