code: def process_doc(source, labels, pipe):
      """
      Given a source text and a list of labels, tokenize the text, then assign labels based on the spans defined
      """
      doc = pipe(source)
      sentences = doc.sentences
      for sentence in sentences:
          for token in sentence.tokens:
              token.ner = "O"
  
      for label in labels:
          ner = label["label"]
          start_offset = label["startOffset"]
          end_offset = label["endOffset"]
          for sentence in sentences:
              if (
                  sentence.tokens[0].start_char <= start_offset
                  and sentence.tokens[-1].end_char >= end_offset
              ):
                  # found the sentence!
                  break
          else:  # for-else again!  deal with it
              continue
  
          start_token = None
          end_token = None
          for token_idx, token in enumerate(sentence.tokens):
              if (
                  token.start_char <= start_offset
                  and token.end_char > start_offset
              ):
                  # ideally we'd have start_char == start_offset, but maybe our
                  # tokenization doesn't match the tokenization of the annotators
                  start_token = token
                  start_token.ner = "B-" + ner
              elif start_token is not None:
                  if token.start_char >= end_offset and token_idx > 0:
                      end_token = sentence.tokens[token_idx - 1]
                      break
                  if (
                      token.end_char == end_offset
                      and token_idx > 0
                      and token.text in (",", ".")
                  ):
                      end_token = sentence.tokens[token_idx - 1]
                      break
                  token.ner = "I-" + ner
              if token.end_char >= end_offset and end_token is None:
                  end_token = token
                  break
          if start_token is None or end_token is None:
              raise AssertionError("This should not happen")
  
      return [
          [(token.text, token.ner) for token in sentence.tokens]
          for sentence in sentences
      ]
  
file_path: stanza/stanza/utils/datasets/ner/convert_amt.py
imports: import argparse
  import copy
  import json
  from operator import itemgetter
  import sys
  from tqdm import tqdm
  import stanza
  from stanza.utils.datasets.ner.utils import write_sentences
  import stanza.utils.datasets.ner.prepare_ner_file
label_desc: Use image node_id 5 for calling a global function with example usage: process_doc(source, labels, pipe) and returns: unknown