code: def char_span_to_token_span(token_offsets, character_span):
      """
      Converts a character span from a passage into the corresponding token span in the tokenized
      version of the passage.  If you pass in a character span that does not correspond to complete
      tokens in the tokenized version, we'll do our best, but the behavior is officially undefined.
      We return an error flag in this case, and have some debug logging so you can figure out the
      cause of this issue (in SQuAD, these are mostly either tokenization problems or annotation
      problems; there's a fair amount of both).
  
      The basic outline of this method is to find the token span that has the same offsets as the
      input character span.  If the tokenizer tokenized the passage correctly and has matching
      offsets, this is easy.  We try to be a little smart about cases where they don't match exactly,
      but mostly just find the closest thing we can.
  
      The returned ``(begin, end)`` indices are `inclusive` for both ``begin`` and ``end``.
      So, for example, ``(2, 2)`` is the one word span beginning at token index 2, ``(3, 4)`` is the
      two-word span beginning at token index 3, and so on.
  
      Returns
      -------
      token_span : ``Tuple[int, int]``
          `Inclusive` span start and end token indices that match as closely as possible to the input
          character spans.
      error : ``bool``
          Whether the token spans match the input character spans exactly.  If this is ``False``, it
          means there was an error in either the tokenization or the annotated character span.
      """
      # We have token offsets into the passage from the tokenizer; we _should_ be able to just find
      # the tokens that have the same offsets as our span.
      error = False
      start_index = 0
      while (
          start_index < len(token_offsets)
          and token_offsets[start_index][0] < character_span[0]
      ):
          start_index += 1
      # start_index should now be pointing at the span start index.
      if token_offsets[start_index][0] > character_span[0]:
          # In this case, a tokenization or labeling issue made us go too far - the character span
          # we're looking for actually starts in the previous token.  We'll back up one.
          logger.debug(
              "Bad labelling or tokenization - start offset doesn't match"
          )
          start_index -= 1
      if token_offsets[start_index][0] != character_span[0]:
          error = True
      end_index = start_index
      while (
          end_index < len(token_offsets)
          and token_offsets[end_index][1] < character_span[1]
      ):
          end_index += 1
      if (
          end_index == start_index
          and token_offsets[end_index][1] > character_span[1]
      ):
          # Looks like there was a token that should have been split, like "1854-1855", where the
          # answer is "1854".  We can't do much in this case, except keep the answer as the whole
          # token.
          logger.debug("Bad tokenization - end offset doesn't match")
      elif token_offsets[end_index][1] > character_span[1]:
          # This is a case where the given answer span is more than one token, and the last token is
          # cut off for some reason, like "split with Luckett and Rober", when the original passage
          # said "split with Luckett and Roberson".  In this case, we'll just keep the end index
          # where it is, and assume the intent was to mark the whole token.
          logger.debug(
              "Bad labelling or tokenization - end offset doesn't match"
          )
      if token_offsets[end_index][1] != character_span[1]:
          error = True
      return (start_index, end_index), error
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/dataset_readers/reading_comprehension/util.py
imports: from __future__ import absolute_import
  from collections import Counter, defaultdict
  import logging
  import string
  from allennlp.data.fields import Field, TextField, IndexField, MetadataField
  from allennlp.data.instance import Instance
  from allennlp.data.token_indexers import TokenIndexer
  from allennlp.data.tokenizers import Token
label_desc: Use image node_id 2 for calling a global function with example usage: char_span_to_token_span(token_offsets, character_span) and returns: start_index,  end_index,  error