text: # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  
  """create squad data"""
  
  import collections
  import json
  import tokenization
  
  
  class SquadExample():
      """extract column contents from raw data"""
      def __init__(self,
                   qas_id,
                   question_text,
                   doc_tokens,
                   orig_answer_text=None,
                   start_position=None,
                   end_position=None,
                   is_impossible=False):
          self.qas_id = qas_id
          self.question_text = question_text
          self.doc_tokens = doc_tokens
          self.orig_answer_text = orig_answer_text
          self.start_position = start_position
          self.end_position = end_position
          self.is_impossible = is_impossible
  
  
  class InputFeatures():
      """A single set of features of data."""
      def __init__(self,
                   unique_id,
                   example_index,
                   doc_span_index,
                   tokens,
                   token_to_orig_map,
                   token_is_max_context,
                   input_ids,
                   input_mask,
                   segment_ids,
                   start_position=None,
                   end_position=None,
                   is_impossible=None):
          self.unique_id = unique_id
          self.example_index = example_index
          self.doc_span_index = doc_span_index
          self.tokens = tokens
          self.token_to_orig_map = token_to_orig_map
          self.token_is_max_context = token_is_max_context
          self.input_ids = input_ids
          self.input_mask = input_mask
          self.segment_ids = segment_ids
          self.start_position = start_position
          self.end_position = end_position
          self.is_impossible = is_impossible
  
  
  def read_squad_examples(input_file, is_training, version_2_with_negative=False):
      """Return list of SquadExample from input_data or input_file (SQuAD json file)"""
      with open(input_file, "r") as reader:
          input_data = json.load(reader)["data"]
  
      def is_whitespace(c):
          if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
              return True
          return False
  
      def token_offset(text):
          doc_tokens = []
          char_to_word_offset = []
          prev_is_whitespace = True
          for c in paragraph_text:
              if is_whitespace(c):
                  prev_is_whitespace = True
              else:
                  if prev_is_whitespace:
                      doc_tokens.append(c)
                  else:
                      doc_tokens[-1] += c
                  prev_is_whitespace = False
              char_to_word_offset.append(len(doc_tokens) - 1)
          return (doc_tokens, char_to_word_offset)
  
  
      def process_one_example(qa, is_training, version_2_with_negative, doc_tokens, char_to_word_offset):
          qas_id = qa["id"]
          question_text = qa["question"]
          start_position = -1
          end_position = -1
          orig_answer_text = ""
          is_impossible = False
          if is_training:
              if version_2_with_negative:
                  is_impossible = qa["is_impossible"]
              if (len(qa["answers"]) != 1) and (not is_impossible):
                  raise ValueError("For training, each question should have exactly 1 answer.")
              if not is_impossible:
                  answer = qa["answers"][0]
                  orig_answer_text = answer["text"]
                  answer_offset = answer["answer_start"]
                  answer_length = len(orig_answer_text)
                  start_position = char_to_word_offset[answer_offset]
                  end_position = char_to_word_offset[answer_offset + answer_length - 1]
                  actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
                  cleaned_answer_text = " ".join(tokenization.whitespace_tokenize(orig_answer_text))
                  if actual_text.find(cleaned_answer_text) == -1:
                      return None
          example = SquadExample(
              qas_id=qas_id,
              question_text=question_text,
              doc_tokens=doc_tokens,
              orig_answer_text=orig_answer_text,
              start_position=start_position,
              end_position=end_position,
              is_impossible=is_impossible)
          return example
  
  
      examples = []
      for entry in input_data:
          for paragraph in entry["paragraphs"]:
              paragraph_text = paragraph["context"]
              doc_tokens, char_to_word_offset = token_offset(paragraph_text)
              for qa in paragraph["qas"]:
                  one_example = process_one_example(qa, is_training, version_2_with_negative,
                                                    doc_tokens, char_to_word_offset)
                  if one_example is not None:
                      examples.append(one_example)
      return examples
  
  def _check_is_max_context(doc_spans, cur_span_index, position):
      """Check if this is the 'max context' doc span for the token."""
      best_score = None
      best_span_index = None
      for (span_index, doc_span) in enumerate(doc_spans):
          end = doc_span.start + doc_span.length - 1
          if position < doc_span.start:
              continue
          if position > end:
              continue
          num_left_context = position - doc_span.start
          num_right_context = end - position
          score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
          if best_score is None or score > best_score:
              best_score = score
              best_span_index = span_index
      return cur_span_index == best_span_index
  
  def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer,
                           orig_answer_text):
      """Returns tokenized answer spans that better match the annotated answer."""
      tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text))
  
      for new_start in range(input_start, input_end + 1):
          for new_end in range(input_end, new_start - 1, -1):
              text_span = " ".join(doc_tokens[new_start:(new_end + 1)])
              if text_span == tok_answer_text:
                  return (new_start, new_end)
  
      return (input_start, input_end)
  
  
  def convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride,
                                   max_query_length, is_training, output_fn, vocab_file):
      """Loads a data file into a list of `InputBatch`s."""
      unique_id = 1000000000
      output = []
      for (example_index, example) in enumerate(examples):
          query_tokens = tokenizer.tokenize(example.question_text)
  
          if len(query_tokens) > max_query_length:
              query_tokens = query_tokens[0:max_query_length]
  
          tok_to_orig_index = []
          orig_to_tok_index = []
          all_doc_tokens = []
          for (i, token) in enumerate(example.doc_tokens):
              orig_to_tok_index.append(len(all_doc_tokens))
              sub_tokens = tokenizer.tokenize(token)
              for sub_token in sub_tokens:
                  tok_to_orig_index.append(i)
                  all_doc_tokens.append(sub_token)
  
          tok_start_position = None
          tok_end_position = None
          if is_training and example.is_impossible:
              tok_start_position = -1
              tok_end_position = -1
          if is_training and not example.is_impossible:
              tok_start_position = orig_to_tok_index[example.start_position]
              if example.end_position < len(example.doc_tokens) - 1:
                  tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
              else:
                  tok_end_position = len(all_doc_tokens) - 1
              (tok_start_position, tok_end_position) = _improve_answer_span(
                  all_doc_tokens, tok_start_position, tok_end_position, tokenizer,
                  example.orig_answer_text)
  
          # The -3 accounts for [CLS], [SEP] and [SEP]
          max_tokens_for_doc = max_seq_length - len(query_tokens) - 3
  
          # We can have documents that are longer than the maximum sequence length.
          # To deal with this we do a sliding window approach, where we take chunks
          # of the up to our max length with a stride of `doc_stride`.
          _DocSpan = collections.namedtuple("DocSpan", ["start", "length"])
          doc_spans = []
          start_offset = 0
          while start_offset < len(all_doc_tokens):
              length = len(all_doc_tokens) - start_offset
              if length > max_tokens_for_doc:
                  length = max_tokens_for_doc
              doc_spans.append(_DocSpan(start=start_offset, length=length))
              if start_offset + length == len(all_doc_tokens):
                  break
              start_offset += min(length, doc_stride)
  
          for (doc_span_index, doc_span) in enumerate(doc_spans):
              tokens = []
              token_to_orig_map = {}
              token_is_max_context = {}
              segment_ids = []
              tokens.append("[CLS]")
              segment_ids.append(0)
              for token in query_tokens:
                  tokens.append(token)
                  segment_ids.append(0)
              tokens.append("[SEP]")
              segment_ids.append(0)
  
              for i in range(doc_span.length):
                  split_token_index = doc_span.start + i
                  token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
  
                  is_max_context = _check_is_max_context(doc_spans, doc_span_index, split_token_index)
                  token_is_max_context[len(tokens)] = is_max_context
                  tokens.append(all_doc_tokens[split_token_index])
                  segment_ids.append(1)
              tokens.append("[SEP]")
              segment_ids.append(1)
  
              input_ids = tokenization.convert_tokens_to_ids(vocab_file, tokens)
  
              # The mask has 1 for real tokens and 0 for padding tokens. Only real
              # tokens are attended to.
              input_mask = [1] * len(input_ids)
  
              # Zero-pad up to the sequence length.
              while len(input_ids) < max_seq_length:
                  input_ids.append(0)
                  input_mask.append(0)
                  segment_ids.append(0)
  
              assert len(input_ids) == max_seq_length
              assert len(input_mask) == max_seq_length
              assert len(segment_ids) == max_seq_length
  
              start_position = None
              end_position = None
              if is_training and not example.is_impossible:
                  # For training, if our document chunk does not contain an annotation
                  # we throw it out, since there is nothing to predict.
                  doc_start = doc_span.start
                  doc_end = doc_span.start + doc_span.length - 1
                  out_of_span = False
                  if not (tok_start_position >= doc_start and tok_end_position <= doc_end):
                      out_of_span = True
                  if out_of_span:
                      start_position = 0
                      end_position = 0
                  else:
                      doc_offset = len(query_tokens) + 2
                      start_position = tok_start_position - doc_start + doc_offset
                      end_position = tok_end_position - doc_start + doc_offset
  
              if is_training and example.is_impossible:
                  start_position = 0
                  end_position = 0
  
              feature = InputFeatures(
                  unique_id=unique_id,
                  example_index=example_index,
                  doc_span_index=doc_span_index,
                  tokens=tokens,
                  token_to_orig_map=token_to_orig_map,
                  token_is_max_context=token_is_max_context,
                  input_ids=input_ids,
                  input_mask=input_mask,
                  segment_ids=segment_ids,
                  start_position=start_position,
                  end_position=end_position,
                  is_impossible=example.is_impossible)
  
              # Run callback
              output.append(feature)
              unique_id += 1
      return output
  