code: def find_valid_answer_spans(passage_tokens, answer_texts):
      """
      Finds a list of token spans in ``passage_tokens`` that match the given ``answer_texts``.  This
      tries to find all spans that would evaluate to correct given the SQuAD and TriviaQA official
      evaluation scripts, which do some normalization of the input text.
  
      Note that this could return duplicate spans!  The caller is expected to be able to handle
      possible duplicates (as already happens in the SQuAD dev set, for instance).
      """
      normalized_tokens = [
          token.text.lower().strip(STRIPPED_CHARACTERS)
          for token in passage_tokens
      ]
      # Because there could be many `answer_texts`, we'll do the most expensive pre-processing
      # step once.  This gives us a map from tokens to the position in the passage they appear.
      word_positions = defaultdict(list)
      for i, token in enumerate(normalized_tokens):
          word_positions[token].append(i)
      spans = []
      for answer_text in answer_texts:
          # For each answer, we'll first find all valid start positions in the passage.  Then
          # we'll grow each span to the same length as the number of answer tokens, and see if we
          # have a match.  We're a little tricky as we grow the span, skipping words that are
          # already pruned from the normalized answer text, and stopping early if we don't match.
          answer_tokens = (
              answer_text.lower().strip(STRIPPED_CHARACTERS).split()
          )
          num_answer_tokens = len(answer_tokens)
          for span_start in word_positions[answer_tokens[0]]:
              span_end = span_start  # span_end is _inclusive_
              answer_index = 1
              while (
                  answer_index < num_answer_tokens
                  and span_end + 1 < len(normalized_tokens)
              ):
                  token = normalized_tokens[span_end + 1]
                  if answer_tokens[answer_index] == token:
                      answer_index += 1
                      span_end += 1
                  elif token in IGNORED_TOKENS:
                      span_end += 1
                  else:
                      break
              if num_answer_tokens == answer_index:
                  spans.append((span_start, span_end))
      return spans
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/dataset_readers/reading_comprehension/util.py
imports: from __future__ import absolute_import
  from collections import Counter, defaultdict
  import logging
  import string
  from allennlp.data.fields import Field, TextField, IndexField, MetadataField
  from allennlp.data.instance import Instance
  from allennlp.data.token_indexers import TokenIndexer
  from allennlp.data.tokenizers import Token
label_desc: Use image node_id 3 for calling a global function with example usage: find_valid_answer_spans(passage_tokens, answer_texts) and returns: spans